diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..44e3161 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,153 @@ +name: NoCodeML CI + +on: + push: + branches: + - main + - release/v3-revival + pull_request: + branches: + - main + +concurrency: + group: nocodeml-ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + frontend: + name: Frontend build and lint + runs-on: ubuntu-latest + defaults: + run: + working-directory: Frontend + steps: + - name: Checkout + uses: actions/checkout@v7 + - name: Setup Node + uses: actions/setup-node@v7 + with: + node-version: '24' + cache: npm + cache-dependency-path: Frontend/package-lock.json + - name: Install dependencies + run: npm ci + - name: Typecheck + run: npx tsc -p tsconfig.app.json --noEmit + - name: Build + run: npm run build + env: + VITE_API_URL: http://localhost:8000 + - name: Lint + run: npm run lint + - name: Audit critical dependency vulnerabilities + run: npm audit --audit-level=critical + + backend: + name: Guest backend tests + runs-on: ubuntu-latest + defaults: + run: + working-directory: Backend + env: + PYTHONPATH: . + SESSION_ROOT_DIR: /tmp/nocodeml-ci-sessions + CELERY_BROKER_URL: memory:// + CELERY_RESULT_BACKEND: cache+memory:// + steps: + - name: Checkout + uses: actions/checkout@v7 + - name: Setup Python + uses: actions/setup-python@v7 + with: + python-version: '3.11' + cache: pip + cache-dependency-path: Backend/requirements.txt + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + python -m pip install pytest + - name: Compile backend + run: python -m compileall -q app + - name: Run guest, ML and isolation tests + run: python -m pytest -q + + backend-container: + name: Production guest container + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v7 + - name: Validate deployment files + run: | + python -m json.tool vercel.json >/dev/null + docker compose --env-file deploy/oci/.env.example -f deploy/oci/docker-compose.yml config >/dev/null + - name: Build backend image + run: docker build --pull -t nocodeml-backend:ci Backend + - name: Verify non-root runtime user + run: test "$(docker image inspect nocodeml-backend:ci --format '{{.Config.User}}')" = "nocodeml" + - name: Smoke-test database-free container + run: | + docker run -d --rm --name nocodeml-ci -p 8000:8000 \ + -e SESSION_ROOT_DIR=/tmp/nocodeml-sessions \ + nocodeml-backend:ci + for attempt in $(seq 1 30); do + if curl --fail --silent http://127.0.0.1:8000/health >/dev/null; then + break + fi + sleep 2 + done + curl --fail http://127.0.0.1:8000/health + curl --fail http://127.0.0.1:8000/ready + curl --fail -X POST http://127.0.0.1:8000/api/v1/session + docker stop nocodeml-ci + + backend-arm64: + name: ARM64 ML compatibility + runs-on: ubuntu-24.04-arm + defaults: + run: + working-directory: Backend + env: + PYTHONPATH: . + SESSION_ROOT_DIR: /tmp/nocodeml-arm64-sessions + CELERY_BROKER_URL: memory:// + CELERY_RESULT_BACKEND: cache+memory:// + steps: + - name: Checkout + uses: actions/checkout@v7 + - name: Setup Python + uses: actions/setup-python@v7 + with: + python-version: '3.11' + cache: pip + cache-dependency-path: Backend/requirements.txt + - name: Install ML stack + run: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + python -m pip install pytest + - name: Verify native ARM64 libraries + run: | + python - <<'PY' + import platform + import lightgbm + import numpy + import pandas + import sklearn + import xgboost + + machine = platform.machine().lower() + assert machine in {'aarch64', 'arm64'}, f'Expected ARM64 runner, got {machine}' + print('architecture:', machine) + print('numpy:', numpy.__version__) + print('pandas:', pandas.__version__) + print('scikit-learn:', sklearn.__version__) + print('xgboost:', xgboost.__version__) + print('lightgbm:', lightgbm.__version__) + PY + - name: Run real ML compatibility tests + run: python -m pytest -q tests/test_ml_pipeline.py tests/test_workspace_training.py tests/test_eda_identity_detection.py diff --git a/.gitignore b/.gitignore index cdee88d..ccc7b43 100644 --- a/.gitignore +++ b/.gitignore @@ -2,39 +2,31 @@ # NoCodeML Platform - Git Ignore Configuration # ============================================ -# ============================================ -# Documentation and MD Files (Keep only README.md) -# ============================================ -# Exclude all documentation folders +# Documentation: keep release-facing docs, ignore scratch documentation. docs/ Backend/docs/ Frontend/docs/ Dummy/docs/ - -# Exclude Frontend documentation folder specifically Frontend/docs/** - -# Exclude all markdown files except root README.md *.md !README.md -# Re-exclude README.md in subdirectories +!DEPLOYMENT.md Backend/README.md Frontend/README.md Dummy/README.md -# Keep important files +# Keep important project safety/docs files +!AGENTS.md +!SUPABASE_HUB_RULES.md !LICENSE # ============================================ # Python / Backend # ============================================ -# Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class *.so - -# Distribution / packaging .Python build/ develop-eggs/ @@ -53,12 +45,8 @@ share/python-wheels/ .installed.cfg *.egg MANIFEST - -# PyInstaller *.manifest *.spec - -# Unit test / coverage reports htmlcov/ .tox/ .nox/ @@ -71,104 +59,62 @@ coverage.xml *.py,cover .hypothesis/ .pytest_cache/ - -# Translations *.mo *.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: instance/ .webassets-cache - -# Scrapy stuff: .scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython +.ipynb_checkpoints/ profile_default/ ipython_config.py - -# pyenv .python-version - -# pipenv Pipfile.lock - -# PEP 582 __pypackages__/ - -# Celery stuff celerybeat-schedule celerybeat.pid - -# SageMath parsed files *.sage.py -# Environments +# Environments and secrets: ignore every real env variant. .env -.venv +.env.* +*.env +*.env.* +.venv/ env/ venv/ ENV/ env.bak/ venv.bak/ -# Keep example env files +# Explicitly keep templates only. !.env.example +!.env.production.example !env.example +!*.env.example -# Spyder project settings .spyderproject .spyproject - -# Rope project settings .ropeproject - -# mkdocs documentation /site - -# mypy .mypy_cache/ .dmypy.json dmypy.json - -# Pyre type checker .pyre/ # ============================================ # Node.js / Frontend # ============================================ -# Dependencies node_modules/ jspm_packages/ - -# Package manager files npm-debug.log* yarn-debug.log* yarn-error.log* lerna-debug.log* pnpm-debug.log* - -# Build outputs dist/ dist-ssr/ *.local - -# Editor directories and files .vscode/* !.vscode/extensions.json .idea/ @@ -178,14 +124,11 @@ dist-ssr/ *.njsproj *.sln *.sw? - -# Bun .bun/ # ============================================ # Docker # ============================================ -# Docker volumes and data docker-data/ postgres-data/ redis-data/ @@ -193,24 +136,16 @@ redis-data/ # ============================================ # Database # ============================================ -# Database files *.db *.sqlite *.sqlite3 -# Alembic -# Keep versions folder but ignore specific migration files if needed -# alembic/versions/*.py - # ============================================ # ML Models and Data # ============================================ -# Uploaded datasets uploads/ Backend/uploads/ app/uploads/ - -# Trained models models/ !Backend/app/models/ !Backend/app/models/** @@ -221,8 +156,6 @@ models/ *.ckpt *.joblib *.model - -# Large data files *.csv *.xlsx *.xls @@ -233,28 +166,24 @@ models/ !tsconfig.json !tsconfig.*.json !components.json -!bun.lockb +!vercel.json +!Frontend/vercel.json # ============================================ # Operating System # ============================================ -# macOS .DS_Store .AppleDouble .LSOverride ._* - -# Windows Thumbs.db Thumbs.db:encryptable ehthumbs.db -ehthumbs_vista.db +ehthumbs.vista.db *.stackdump [Dd]esktop.ini $RECYCLE.BIN/ *.lnk - -# Linux *~ .fuse_hidden* .directory @@ -264,26 +193,19 @@ $RECYCLE.BIN/ # ============================================ # IDEs and Editors # ============================================ -# VSCode .vscode/ !.vscode/settings.json !.vscode/tasks.json !.vscode/launch.json !.vscode/extensions.json *.code-workspace - -# JetBrains IDEs .idea/ *.iml *.iws *.ipr out/ - -# Sublime Text *.sublime-project *.sublime-workspace - -# Vim [._]*.s[a-v][a-z] [._]*.sw[a-p] [._]s[a-rt-v][a-z] @@ -291,8 +213,6 @@ out/ [._]sw[a-p] Session.vim Sessionx.vim - -# Emacs *~ \#*\# /.emacs.desktop @@ -309,8 +229,6 @@ Sessionx.vim *.swo *~.nib *.log - -# Backup and old files *_OLD.tsx *_OLD.ts *_OLD.jsx @@ -325,29 +243,15 @@ Sessionx.vim # ============================================ # Project Specific # ============================================ -# Exclude dummy folder Dummy/ - -# Exclude zip files and PDFs *.zip *.pdf - -# Test files test_*.json *_test.json - -# Feature lists and phase reports COMPLETE_FEATURES_LIST.md FEATURES_LIST.md PHASE*.md DOCUMENTATION_CLEANUP_REPORT.md README_OLD.md - -# Screenshots - Allow folder but initially ignore images (add them manually) -# You can comment out these lines after adding your screenshots -# screenshots/*.png -# screenshots/*.jpg -# screenshots/*.jpeg -# But always keep the README and .gitkeep !screenshots/README.md !screenshots/.gitkeep diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..ffee97e --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,38 @@ +# NoCodeML Agent Instructions + +## Project identity + +- Application: NoCodeML +- Repository: `Rishikeshsanin/NoCodeML` +- Project Hub app slug: `nocodeml` +- Project Hub schema: `nocodeml` + +## Absolute isolation rule + +NoCodeML may read, write, migrate, test, or deploy only resources explicitly registered to the `nocodeml` application scope. + +Never inspect, query, alter, migrate, truncate, delete, or depend on another application's schema, tables, functions, storage buckets, secrets, credentials, queues, or deployment configuration. + +## Database rules + +- Read `hub.read_me_first` before any Project Hub write. +- Verify the `hub.apps` registry entry for `nocodeml` before database work. +- Require `slug = schema_name = 'nocodeml'`. +- Run `hub.assert_app_scope('nocodeml', 'nocodeml')` before app database changes. +- Use fully-qualified names such as `nocodeml.datasets`. +- Do not create NoCodeML application tables in `public`. +- Do not modify `hub`, `auth`, `storage`, `realtime`, `public`, or any other application schema from NoCodeML work. +- Do not create cross-application foreign keys or dependencies. +- Keep user-facing tables protected with appropriate access controls. +- Keep secrets out of Git and browser bundles. + +## Deployment rules + +- Frontend public configuration may contain only non-secret values. +- Backend/database credentials are server-only. +- Never expose Project Hub administrative/service-role credentials to the browser or ordinary application code. +- Deployment must preserve NoCodeML's schema isolation. + +## Change safety + +If a requested change could affect another Project Hub application or shared project-wide infrastructure, stop that change until its impact is explicitly reviewed. diff --git a/Backend/.dockerignore b/Backend/.dockerignore index f4ca7f4..6481d80 100644 --- a/Backend/.dockerignore +++ b/Backend/.dockerignore @@ -8,7 +8,15 @@ __pycache__ *.egg-info dist build + +# Never send real environment/secrets files to the Docker build context. .env +.env.* +*.env +*.env.* +!.env.example +!.env.production.example + .venv venv/ .git @@ -23,4 +31,4 @@ README.md *.log .pytest_cache .coverage -htmlcov/ \ No newline at end of file +htmlcov/ diff --git a/Backend/.env.example b/Backend/.env.example index 729fb1a..7b848fe 100644 --- a/Backend/.env.example +++ b/Backend/.env.example @@ -1,35 +1,27 @@ -# =========================================== -# NoCodeML Backend - Environment Configuration -# =========================================== -# Copy this file to .env and update the values +# NoCodeML V3 Backend Environment -# --- PostgreSQL Database Configuration --- -# These values are used by the 'postgres' service in docker-compose.yml -POSTGRES_USER=myuser -POSTGRES_PASSWORD=mysecretpassword -POSTGRES_DB=nocodeml_db +# Runtime +ENVIRONMENT=development +PROJECT_NAME=NoCodeML API -# --- Application Configuration --- -# Full database connection string for SQLAlchemy (Async) -# Format: postgresql+://:@:/ -# Use 'postgres' as the host when running with docker-compose -# Use 'localhost' when running backend outside of docker -DATABASE_URL=postgresql+psycopg://myuser:mysecretpassword@postgres:5432/nocodeml_db +# Temporary guest workspaces +# Visitor datasets, generated models, predictions and exports live only here. +SESSION_ROOT_DIR=/tmp/nocodeml-sessions +SESSION_TTL_MINUTES=60 +SESSION_CLEANUP_INTERVAL_SECONDS=300 +SESSION_CLOSE_GRACE_SECONDS=30 -# --- Celery & Redis Configuration --- -# Connection strings for Celery to use Redis -# Use 'redis' as the host when running with docker-compose -# Use 'localhost' when running backend outside of docker -CELERY_BROKER_URL=redis://redis:6379/0 -CELERY_RESULT_BACKEND=redis://redis:6379/0 +# Bounded in-process ML training +WORKSPACE_TRAINING_WORKERS=1 +WORKSPACE_MAX_MODELS_PER_RUN=8 -# --- Security Configuration --- -# Secret key for signing JWTs -# IMPORTANT: Change this to a long, random string for production -# Generate with: openssl rand -hex 32 -SECRET_KEY=change-this-to-a-secure-random-string-in-production +# Allowed frontend origins (comma-separated, no trailing slash) +BACKEND_CORS_ORIGINS=http://localhost:5173,http://127.0.0.1:5173 -# --- Optional Configuration --- -# PROJECT_NAME=NoCodeML API -# DEBUG=False -# ALLOWED_ORIGINS=http://localhost:5173,http://localhost:3000 +# Optional Data Science Assistant (server-side only) +# Raw dataset rows are not injected into assistant requests by the backend. +GEMINI_API_KEY= +GEMINI_MODEL=gemini-3.7-flash + +# No DATABASE_URL, PostgreSQL, Supabase, Redis or Celery service is required +# for the public guest workflow in NoCodeML V3. diff --git a/Backend/.env.production.example b/Backend/.env.production.example new file mode 100644 index 0000000..117a027 --- /dev/null +++ b/Backend/.env.production.example @@ -0,0 +1,48 @@ +# NoCodeML V3 production environment template +# Copy to Backend/.env.production on the deployment host and replace placeholders. +# NEVER commit the real .env.production file. + +ENVIRONMENT=production +PROJECT_NAME=NoCodeML API + +# Supabase Project Hub PostgreSQL connection (server-side only). +# Use the NoCodeML database credential/connection and keep DB_SCHEMA exactly nocodeml. +DATABASE_URL=postgresql+psycopg://USER:PASSWORD@HOST:PORT/postgres?sslmode=require +DB_SCHEMA=nocodeml + +# JWT signing secret. Generate at least 32 random bytes, for example: +# openssl rand -hex 32 +SECRET_KEY=REPLACE_WITH_A_STRONG_RANDOM_SECRET +ACCESS_TOKEN_EXPIRE_MINUTES=60 + +# Public Vercel/frontend origin(s), comma-separated. +BACKEND_CORS_ORIGINS=https://REPLACE_WITH_FRONTEND.vercel.app + +# Public backend hostname used by the Caddy container. +# A normal domain is fine. For a free VM without a domain, an IP-embedded +# sslip.io/nip.io hostname can be used after the VM receives its public IPv4. +PUBLIC_HOSTNAME=REPLACE_WITH_BACKEND_HOSTNAME + +# Redis is private inside Docker Compose and is not exposed publicly. +CELERY_BROKER_URL=redis://redis:6379/0 +CELERY_RESULT_BACKEND=redis://redis:6379/0 + +# Single-VM free deployment: API and worker share the same private Docker volume. +ARTIFACT_STORAGE_BACKEND=local +DATASETS_DIR=/data/datasets +MODELS_DIR=/data/models +PREDICTIONS_DIR=/data/predictions +ARTIFACT_CACHE_DIR=/tmp/nocodeml-artifacts + +# Optional Data Science Assistant. Leave blank to disable safely. +GEMINI_API_KEY= +GEMINI_MODEL=gemini-3.7-flash + +# Optional S3-compatible storage settings. These are ignored while +# ARTIFACT_STORAGE_BACKEND=local. +S3_ENDPOINT_URL= +S3_ACCESS_KEY_ID= +S3_SECRET_ACCESS_KEY= +S3_BUCKET_NAME= +S3_REGION=auto +S3_ADDRESSING_STYLE=path diff --git a/Backend/Caddyfile b/Backend/Caddyfile new file mode 100644 index 0000000..e19fa72 --- /dev/null +++ b/Backend/Caddyfile @@ -0,0 +1,13 @@ +{$PUBLIC_HOSTNAME} { + encode zstd gzip + + header { + X-Content-Type-Options "nosniff" + X-Frame-Options "DENY" + Referrer-Policy "strict-origin-when-cross-origin" + Permissions-Policy "camera=(), microphone=(), geolocation=()" + -Server + } + + reverse_proxy api:8000 +} diff --git a/Backend/Dockerfile b/Backend/Dockerfile index a400c8a..ffe2202 100644 --- a/Backend/Dockerfile +++ b/Backend/Dockerfile @@ -1,47 +1,38 @@ -# Builder stage -FROM python:3.11-slim as builder +# syntax=docker/dockerfile:1 +FROM python:3.11-slim AS builder WORKDIR /usr/src/app +ENV PIP_DISABLE_PIP_VERSION_CHECK=1 PIP_NO_CACHE_DIR=1 -# Install system dependencies -RUN apt-get update && apt-get install -y --no-install-recommends \ - gcc \ - libpq-dev \ +RUN apt-get update && apt-get install -y --no-install-recommends gcc libpq-dev \ && rm -rf /var/lib/apt/lists/* - -# Copy and install Python dependencies COPY requirements.txt . -RUN pip wheel --no-cache-dir --no-deps --wheel-dir /usr/src/app/wheels -r requirements.txt - -# Runtime stage -FROM python:3.11-slim +RUN pip wheel --wheel-dir /usr/src/app/wheels -r requirements.txt +FROM python:3.11-slim AS runtime WORKDIR /app -# Install runtime dependencies -# libgomp1 is required for LightGBM and other ML libraries -RUN apt-get update && apt-get install -y --no-install-recommends \ - libpq5 \ - libgomp1 \ - && rm -rf /var/lib/apt/lists/* +ENV PYTHONPATH=/app \ + PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + SESSION_ROOT_DIR=/tmp/nocodeml-sessions \ + DATASETS_DIR=/tmp/nocodeml-legacy/datasets \ + MODELS_DIR=/tmp/nocodeml-legacy/models \ + PREDICTIONS_DIR=/tmp/nocodeml-legacy/predictions -# Copy wheels from builder -COPY --from=builder /usr/src/app/wheels /wheels +RUN apt-get update && apt-get install -y --no-install-recommends libpq5 libgomp1 \ + && rm -rf /var/lib/apt/lists/* \ + && groupadd --system --gid 10001 nocodeml \ + && useradd --system --uid 10001 --gid nocodeml --home-dir /app --shell /usr/sbin/nologin nocodeml -# Install Python packages -RUN pip install --no-cache /wheels/* && rm -rf /wheels - -# Copy application code -COPY ./app ./app - -# Copy Alembic configuration and migrations -COPY alembic.ini . -COPY alembic ./alembic +COPY --from=builder /usr/src/app/wheels /wheels +RUN pip install --no-cache-dir /wheels/* && rm -rf /wheels +COPY --chown=nocodeml:nocodeml ./app ./app -# Create necessary directories for data storage -RUN mkdir -p /app/datasets /app/models +USER nocodeml +EXPOSE 8000 -# Set Python path -ENV PYTHONPATH=/app +HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=3)" || exit 1 -CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"] \ No newline at end of file +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000", "--proxy-headers"] diff --git a/Backend/alembic/env.py b/Backend/alembic/env.py index ae8cf66..1709544 100644 --- a/Backend/alembic/env.py +++ b/Backend/alembic/env.py @@ -1,77 +1,63 @@ from logging.config import fileConfig -from sqlalchemy import engine_from_config -from sqlalchemy import pool - from alembic import context +from sqlalchemy import engine_from_config, pool -# Import your models' Base -from app.models import Base from app.core.config import settings +from app.models import Base -# this is the Alembic Config object, which provides -# access to the values within the .ini file in use. -config = context.config -# Override sqlalchemy.url with our actual DATABASE_URL from settings -config.set_main_option('sqlalchemy.url', settings.DATABASE_URL) +config = context.config +config.set_main_option("sqlalchemy.url", settings.DATABASE_URL) -# Interpret the config file for Python logging. -# This line sets up loggers basically. if config.config_file_name is not None: fileConfig(config.config_file_name) -# add your model's MetaData object here -# for 'autogenerate' support target_metadata = Base.metadata -# other values from the config, defined by the needs of env.py, -# can be acquired: -# my_important_option = config.get_main_option("my_important_option") -# ... etc. - - -def run_migrations_offline() -> None: - """Run migrations in 'offline' mode. - This configures the context with just a URL - and not an Engine, though an Engine is acceptable - here as well. By skipping the Engine creation - we don't even need a DBAPI to be available. +def _configure_context(**kwargs) -> None: + context.configure( + target_metadata=target_metadata, + version_table="alembic_version", + version_table_schema=settings.DB_SCHEMA if settings.is_postgres else None, + include_schemas=settings.is_postgres, + compare_type=True, + **kwargs, + ) - Calls to context.execute() here emit the given string to the - script output. - """ +def run_migrations_offline() -> None: url = config.get_main_option("sqlalchemy.url") - context.configure( + _configure_context( url=url, - target_metadata=target_metadata, literal_binds=True, dialect_opts={"paramstyle": "named"}, ) with context.begin_transaction(): + if settings.is_postgres: + context.execute(f'SET search_path TO "{settings.DB_SCHEMA}"') context.run_migrations() def run_migrations_online() -> None: - """Run migrations in 'online' mode. + section = config.get_section(config.config_ini_section, {}) + connect_args = settings.database_connect_args if settings.is_postgres else {} - In this scenario we need to create an Engine - and associate a connection with the context. - - """ connectable = engine_from_config( - config.get_section(config.config_ini_section, {}), + section, prefix="sqlalchemy.", poolclass=pool.NullPool, + connect_args=connect_args, ) with connectable.connect() as connection: - context.configure( - connection=connection, target_metadata=target_metadata - ) + if settings.is_postgres: + connection.exec_driver_sql(f'SET search_path TO "{settings.DB_SCHEMA}"') + connection.commit() + + _configure_context(connection=connection) with context.begin_transaction(): context.run_migrations() diff --git a/Backend/alembic/versions/000_create_users_table.py b/Backend/alembic/versions/000_create_users_table.py new file mode 100644 index 0000000..46ba966 --- /dev/null +++ b/Backend/alembic/versions/000_create_users_table.py @@ -0,0 +1,35 @@ +"""Create users table. + +Revision ID: 000 +Revises: +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +revision: str = "000" +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "users", + sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True), + sa.Column("email", sa.String(length=320), nullable=False), + sa.Column("hashed_password", sa.String(length=255), nullable=False), + sa.Column("is_active", sa.Boolean(), nullable=False, server_default=sa.true()), + sa.Column("is_superuser", sa.Boolean(), nullable=False, server_default=sa.false()), + sa.Column("is_verified", sa.Boolean(), nullable=False, server_default=sa.false()), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=True), + ) + op.create_index("ix_users_email", "users", ["email"], unique=True) + + +def downgrade() -> None: + op.drop_index("ix_users_email", table_name="users") + op.drop_table("users") diff --git a/Backend/alembic/versions/001_create_datasets_table.py b/Backend/alembic/versions/001_create_datasets_table.py index 2d6ba89..b8394eb 100644 --- a/Backend/alembic/versions/001_create_datasets_table.py +++ b/Backend/alembic/versions/001_create_datasets_table.py @@ -1,47 +1,41 @@ -"""Create datasets table +"""Create datasets table. Revision ID: 001 -Revises: +Revises: 000 Create Date: 2025-10-03 12:00:00.000000 - """ from typing import Sequence, Union from alembic import op import sqlalchemy as sa -from sqlalchemy.dialects.postgresql import UUID, JSONB +from sqlalchemy.dialects.postgresql import JSONB, UUID -# revision identifiers, used by Alembic. -revision: str = '001' -down_revision: Union[str, None] = None +revision: str = "001" +down_revision: Union[str, None] = "000" branch_labels: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None def upgrade() -> None: - """Create the datasets table.""" op.create_table( - 'datasets', - sa.Column('id', UUID(as_uuid=True), primary_key=True, nullable=False), - sa.Column('user_id', sa.Integer(), sa.ForeignKey('users.id', ondelete='CASCADE'), nullable=False, index=True), - sa.Column('name', sa.String(255), nullable=False), - sa.Column('description', sa.Text(), nullable=True), - sa.Column('storage_path', sa.String(1024), nullable=False, unique=True), - sa.Column('file_name', sa.String(255), nullable=False), - sa.Column('file_size_bytes', sa.BigInteger(), nullable=False), - sa.Column('row_count', sa.Integer(), nullable=False), - sa.Column('column_count', sa.Integer(), nullable=False), - sa.Column('column_info', JSONB, nullable=False), - sa.Column('created_at', sa.TIMESTAMP(timezone=True), server_default=sa.func.now(), nullable=False), - sa.Column('updated_at', sa.TIMESTAMP(timezone=True), onupdate=sa.func.now(), nullable=True) + "datasets", + sa.Column("id", UUID(as_uuid=True), primary_key=True, nullable=False), + sa.Column("user_id", sa.Integer(), sa.ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True), + sa.Column("name", sa.String(255), nullable=False), + sa.Column("description", sa.Text(), nullable=True), + sa.Column("storage_path", sa.String(1024), nullable=False, unique=True), + sa.Column("file_name", sa.String(255), nullable=False), + sa.Column("file_size_bytes", sa.BigInteger(), nullable=False), + sa.Column("row_count", sa.Integer(), nullable=False), + sa.Column("column_count", sa.Integer(), nullable=False), + sa.Column("column_info", JSONB, nullable=False), + sa.Column("created_at", sa.TIMESTAMP(timezone=True), server_default=sa.func.now(), nullable=False), + sa.Column("updated_at", sa.TIMESTAMP(timezone=True), onupdate=sa.func.now(), nullable=True), ) - - # Create index on id for faster lookups - op.create_index('ix_datasets_id', 'datasets', ['id']) + op.create_index("ix_datasets_id", "datasets", ["id"]) def downgrade() -> None: - """Drop the datasets table.""" - op.drop_index('ix_datasets_id', 'datasets') - op.drop_table('datasets') + op.drop_index("ix_datasets_id", table_name="datasets") + op.drop_table("datasets") diff --git a/Backend/app/api/__init__.py b/Backend/app/api/__init__.py index 9377b9f..eb55724 100644 --- a/Backend/app/api/__init__.py +++ b/Backend/app/api/__init__.py @@ -1,86 +1,14 @@ -"""API routes package. - -Combines all API routers into a single main router. -This router is then included in the FastAPI app with the /api/v1 prefix. -""" +"""Public API surface for the guest-first NoCodeML release.""" from fastapi import APIRouter -from app.api import auth, datasets, experiments, eda, models, training, predictions - -# Create main API router -api_router = APIRouter() - -# Authentication routes -# POST /auth/register - Register new user -# POST /auth/login - Login and get JWT token (accepts form data) -# GET /auth/me - Get current user info (protected) -# POST /auth/logout - Logout (optional, mostly client-side) -api_router.include_router( - auth.router, - prefix="/auth", - tags=["Authentication"] -) -# Dataset routes -# POST /datasets - Upload new dataset -# GET /datasets - List user's datasets -# GET /datasets/{id} - Get dataset details -# GET /datasets/{id}/preview - Preview dataset contents -# PUT /datasets/{id} - Update dataset metadata -# DELETE /datasets/{id} - Delete dataset -api_router.include_router( - datasets.router, - prefix="/datasets", - tags=["Datasets"] -) +from app.api import assistant, models, session, workspace -# Experiment routes -# POST /experiments - Create new experiment -# GET /experiments - List user's experiments -# GET /experiments/{id} - Get experiment details -# PUT /experiments/{id} - Update experiment -# DELETE /experiments/{id} - Delete experiment -# POST /experiments/{id}/duplicate - Duplicate experiment -api_router.include_router( - experiments.router, - prefix="/experiments", - tags=["Experiments"] -) - -# EDA routes -# GET /datasets/{id}/eda - Get comprehensive EDA summary -# POST /datasets/{id}/plot - Generate plot data dynamically -api_router.include_router( - eda.router, - tags=["EDA"] -) - -# ML Models routes -# GET /models - Get all available ML models grouped by task type -# GET /models/{task_type} - Get models for specific task type -api_router.include_router( - models.router, - tags=["ML Models"] -) - -# Training routes -# POST /training/experiments/{id}/train - Start training jobs -# GET /training/jobs/{id} - Get job status -# GET /training/experiments/{id}/jobs - List jobs for experiment -# GET /training/experiments/{id}/status - Get training status overview -# GET /training/results/{id} - Get training result by result ID -# GET /training/results/job/{id} - Get training result by job ID -api_router.include_router( - training.router, - prefix="/training", - tags=["Training"] -) +api_router = APIRouter() -# Prediction routes -# POST /predictions/experiments/{id}/predict/single - Make single prediction -# POST /predictions/experiments/{id}/predict/batch - Make batch predictions -# GET /predictions/download/{id} - Download batch prediction results -api_router.include_router( - predictions.router, - prefix="/predictions", - tags=["Predictions"] -) +# The V3 public runtime intentionally exposes no account, experiment, dataset, +# training-run or prediction persistence routes. Visitor work exists only in +# the isolated temporary workspace owned by the anonymous session token. +api_router.include_router(session.router, prefix="/session", tags=["Temporary Session"]) +api_router.include_router(workspace.router, prefix="/workspace", tags=["Temporary Workspace"]) +api_router.include_router(models.router, tags=["ML Models"]) +api_router.include_router(assistant.router, prefix="/assistant", tags=["AI Assistant"]) diff --git a/Backend/app/api/assistant.py b/Backend/app/api/assistant.py new file mode 100644 index 0000000..96d13e1 --- /dev/null +++ b/Backend/app/api/assistant.py @@ -0,0 +1,87 @@ +"""Server-side Data Science Assistant for temporary guest sessions.""" +from typing import Literal + +import httpx +from fastapi import APIRouter, HTTPException, status +from pydantic import BaseModel, Field + +from app.api.session import SessionToken +from app.core.config import settings +from app.services.session_manager import session_manager + +router = APIRouter() + + +class AssistantMessage(BaseModel): + role: Literal["user", "assistant"] + content: str = Field(min_length=1, max_length=12000) + + +class AssistantChatRequest(BaseModel): + system_prompt: str = Field(min_length=1, max_length=30000) + messages: list[AssistantMessage] = Field(default_factory=list, max_length=20) + + +class AssistantChatResponse(BaseModel): + content: str + model: str + + +def _gemini_contents(messages: list[AssistantMessage]) -> list[dict]: + recent = messages[-20:] + first_user = next((index for index, item in enumerate(recent) if item.role == "user"), None) + if first_user is None: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="A user message is required.") + recent = recent[first_user:] + if recent[-1].role != "user": + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="The conversation must end with a user message.") + return [ + {"role": "model" if message.role == "assistant" else "user", "parts": [{"text": message.content}]} + for message in recent + ] + + +@router.post("/chat", response_model=AssistantChatResponse) +async def chat(request: AssistantChatRequest, token: SessionToken): + # Validate/touch the temporary workspace before any provider request. + session_manager.touch(token) + + if not settings.GEMINI_API_KEY: + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail="AI assistant is not configured on this deployment yet.", + ) + + # The frontend sends derived experiment context only. Raw uploaded rows are + # intentionally not read or injected by this endpoint. + payload = { + "system_instruction": {"parts": [{"text": request.system_prompt}]}, + "contents": _gemini_contents(request.messages), + "generationConfig": { + "maxOutputTokens": 1200, + "thinkingConfig": {"thinkingLevel": "low"}, + }, + } + + url = "https://generativelanguage.googleapis.com/v1beta/models/" f"{settings.GEMINI_MODEL}:generateContent" + try: + async with httpx.AsyncClient(timeout=45.0) as client: + response = await client.post( + url, + headers={"Content-Type": "application/json", "x-goog-api-key": settings.GEMINI_API_KEY}, + json=payload, + ) + except httpx.RequestError as exc: + raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail="AI provider is temporarily unreachable.") from exc + + if response.status_code >= 400: + raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail="AI provider rejected the request.") + + data = response.json() + candidates = data.get("candidates") or [] + parts = candidates[0].get("content", {}).get("parts", []) if candidates else [] + text = "".join(part.get("text", "") for part in parts if not part.get("thought")).strip() + if not text: + raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail="AI provider returned an empty response.") + + return AssistantChatResponse(content=text, model=settings.GEMINI_MODEL) diff --git a/Backend/app/api/auth.py b/Backend/app/api/auth.py index 066ba6f..579df11 100644 --- a/Backend/app/api/auth.py +++ b/Backend/app/api/auth.py @@ -1,91 +1,79 @@ """Authentication routes for user registration and login.""" from fastapi import APIRouter, Depends, HTTPException, status from fastapi.security import OAuth2PasswordRequestForm -from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy import select +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncSession -from app.core.security import hash_password, verify_password, create_access_token from app.core.deps import get_current_active_user +from app.core.security import create_access_token, hash_password, verify_password from app.db.session import get_db from app.models import User -from app.schemas import UserCreate, UserResponse, Token, LoginRequest +from app.schemas import Token, UserCreate, UserResponse router = APIRouter() @router.post("/register", response_model=UserResponse, status_code=status.HTTP_201_CREATED) -async def register( - user_data: UserCreate, - db: AsyncSession = Depends(get_db) -) -> User: - """Register a new user account.""" - result = await db.execute(select(User).where(User.email == user_data.email)) - existing_user = result.scalar_one_or_none() - - if existing_user: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail="Email already registered" - ) - +async def register(user_data: UserCreate, db: AsyncSession = Depends(get_db)) -> User: + """Register a new user account with a normalized email identity.""" + email = str(user_data.email).strip().lower() + result = await db.execute(select(User).where(User.email == email)) + if result.scalar_one_or_none(): + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail="Email already registered") + new_user = User( - email=user_data.email, + email=email, hashed_password=hash_password(user_data.password), is_active=True, is_superuser=False, - is_verified=False + is_verified=False, ) - db.add(new_user) - await db.commit() - await db.refresh(new_user) - + try: + await db.commit() + await db.refresh(new_user) + except IntegrityError as exc: + # The unique index is the final authority if two registrations race. + await db.rollback() + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail="Email already registered") from exc + return new_user @router.post("/login", response_model=Token) async def login( form_data: OAuth2PasswordRequestForm = Depends(), - db: AsyncSession = Depends(get_db) + db: AsyncSession = Depends(get_db), ) -> dict: - """Authenticate user and return JWT access token.""" - result = await db.execute(select(User).where(User.email == form_data.username)) + """Authenticate a user and return a time-limited JWT access token.""" + email = form_data.username.strip().lower() + result = await db.execute(select(User).where(User.email == email)) user = result.scalar_one_or_none() - + if not user or not verify_password(form_data.password, user.hashed_password): raise HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, detail="Incorrect email or password", headers={"WWW-Authenticate": "Bearer"}, ) - + if not user.is_active: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Inactive user account" - ) - - access_token = create_access_token( - data={"sub": user.email, "user_id": user.id} - ) - - return { - "access_token": access_token, - "token_type": "bearer" - } + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Inactive user account") + + access_token = create_access_token(data={"sub": user.email, "user_id": user.id}) + return {"access_token": access_token, "token_type": "bearer"} @router.get("/me", response_model=UserResponse) -async def get_current_user_info( - current_user: User = Depends(get_current_active_user) -) -> User: - """Get current authenticated user's information.""" +async def get_current_user_info(current_user: User = Depends(get_current_active_user)) -> User: return current_user @router.post("/logout", status_code=status.HTTP_200_OK) -async def logout( - current_user: User = Depends(get_current_active_user) -) -> dict: - """Logout endpoint.""" +async def logout(current_user: User = Depends(get_current_active_user)) -> dict: + # Tokens are stateless; the frontend removes the current token on logout. + # This endpoint remains authenticated so a caller cannot report a successful + # server logout for an already-invalid session. + del current_user return {"message": "Successfully logged out"} diff --git a/Backend/app/api/predictions.py b/Backend/app/api/predictions.py index bd76447..73411a2 100644 --- a/Backend/app/api/predictions.py +++ b/Backend/app/api/predictions.py @@ -1,51 +1,44 @@ """Prediction API endpoints.""" -from fastapi import APIRouter, Depends, HTTPException, UploadFile, File -from fastapi.responses import FileResponse -from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy import select from uuid import UUID -from pathlib import Path -from app.core.deps import get_db, get_current_user +from fastapi import APIRouter, Depends, File, HTTPException, UploadFile +from fastapi.responses import FileResponse, RedirectResponse +from sqlalchemy import desc, select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.deps import get_current_user +from app.db.session import get_db from app.models import User from app.models.experiment import Experiment +from app.models.prediction import PredictionBatch +from app.schemas.prediction import BatchPredictionResponse, SinglePredictionRequest, SinglePredictionResponse +from app.services.artifact_store import artifact_store from app.services.prediction_service import PredictionService -from app.schemas.prediction import ( - SinglePredictionRequest, - SinglePredictionResponse, - BatchPredictionResponse -) + router = APIRouter() prediction_service = PredictionService() +async def _owned_experiment(experiment_id: UUID, user_id: int, db: AsyncSession) -> Experiment: + result = await db.execute( + select(Experiment).where(Experiment.id == experiment_id, Experiment.user_id == user_id) + ) + experiment = result.scalar_one_or_none() + if not experiment: + raise HTTPException(status_code=404, detail="Experiment not found") + return experiment + + @router.post("/experiments/{experiment_id}/predict/single", response_model=SinglePredictionResponse) async def predict_single( experiment_id: UUID, request: SinglePredictionRequest, current_user: User = Depends(get_current_user), - db: AsyncSession = Depends(get_db) + db: AsyncSession = Depends(get_db), ): - """Make a single prediction using the best model from an experiment.""" - # Verify experiment ownership - result = await db.execute( - select(Experiment).filter( - Experiment.id == experiment_id, - Experiment.user_id == current_user.id - ) - ) - experiment = result.scalar_one_or_none() - if not experiment: - raise HTTPException(status_code=404, detail="Experiment not found") - - # Make prediction - prediction_result = await prediction_service.predict_single( - experiment_id=experiment_id, - features=request.features - ) - - return prediction_result + await _owned_experiment(experiment_id, current_user.id, db) + return await prediction_service.predict_single(experiment_id=experiment_id, features=request.features) @router.post("/experiments/{experiment_id}/predict/batch", response_model=BatchPredictionResponse) @@ -53,74 +46,49 @@ async def predict_batch( experiment_id: UUID, file: UploadFile = File(...), current_user: User = Depends(get_current_user), - db: AsyncSession = Depends(get_db) + db: AsyncSession = Depends(get_db), ): - """Upload CSV for batch predictions.""" - # Verify experiment ownership - result = await db.execute( - select(Experiment).filter( - Experiment.id == experiment_id, - Experiment.user_id == current_user.id - ) - ) - experiment = result.scalar_one_or_none() - if not experiment: - raise HTTPException(status_code=404, detail="Experiment not found") - - # Validate file type - if not file.filename.endswith('.csv'): - raise HTTPException(status_code=400, detail="Only CSV files are supported") - - # Process batch predictions with user_id for ownership tracking - prediction_result = await prediction_service.predict_batch( + await _owned_experiment(experiment_id, current_user.id, db) + + filename = (file.filename or "").lower() + if not filename.endswith(".csv"): + await file.close() + raise HTTPException(status_code=400, detail="Only CSV files are supported for batch prediction") + + return await prediction_service.predict_batch( experiment_id=experiment_id, file=file, - user_id=current_user.id + user_id=current_user.id, ) - - return prediction_result @router.get("/experiments/{experiment_id}/history") async def get_prediction_history( experiment_id: UUID, current_user: User = Depends(get_current_user), - db: AsyncSession = Depends(get_db) + db: AsyncSession = Depends(get_db), ): - """Get prediction history for an experiment.""" - from app.models.prediction import PredictionBatch - from sqlalchemy import desc - - # Verify experiment ownership - result = await db.execute( - select(Experiment).filter( - Experiment.id == experiment_id, - Experiment.user_id == current_user.id - ) - ) - experiment = result.scalar_one_or_none() - if not experiment: - raise HTTPException(status_code=404, detail="Experiment not found") - - # Get prediction history + await _owned_experiment(experiment_id, current_user.id, db) + result = await db.execute( - select(PredictionBatch).filter( + select(PredictionBatch) + .where( PredictionBatch.experiment_id == experiment_id, - PredictionBatch.user_id == current_user.id - ).order_by(desc(PredictionBatch.created_at)) + PredictionBatch.user_id == current_user.id, + ) + .order_by(desc(PredictionBatch.created_at)) ) predictions = result.scalars().all() - return { "predictions": [ { - "id": str(pred.id), - "experiment_id": str(pred.experiment_id), - "total_predictions": pred.total_predictions, - "created_at": pred.created_at.isoformat(), - "download_url": f"/api/v1/predictions/download/{pred.id}" + "id": str(prediction.id), + "experiment_id": str(prediction.experiment_id), + "total_predictions": prediction.total_predictions, + "created_at": prediction.created_at.isoformat(), + "download_url": f"/api/v1/predictions/download/{prediction.id}", } - for pred in predictions + for prediction in predictions ] } @@ -129,43 +97,36 @@ async def get_prediction_history( async def download_predictions( prediction_id: str, current_user: User = Depends(get_current_user), - db: AsyncSession = Depends(get_db) + db: AsyncSession = Depends(get_db), ): - """Download batch prediction results with ownership validation.""" - from app.models.prediction import PredictionBatch - - # Validate prediction ID format try: pred_uuid = UUID(prediction_id) - except ValueError: - raise HTTPException(status_code=400, detail="Invalid prediction ID format") - - # Check ownership in database + except ValueError as exc: + raise HTTPException(status_code=400, detail="Invalid prediction ID format") from exc + result = await db.execute( - select(PredictionBatch).filter( + select(PredictionBatch).where( PredictionBatch.id == pred_uuid, - PredictionBatch.user_id == current_user.id + PredictionBatch.user_id == current_user.id, ) ) prediction_batch = result.scalar_one_or_none() - if not prediction_batch: - raise HTTPException( - status_code=404, - detail="Prediction results not found or you don't have permission to access them" - ) - - # Get file path from database - file_path = Path(prediction_batch.file_path) - - if not file_path.exists(): - raise HTTPException( - status_code=404, - detail="Prediction file has been deleted or moved" - ) - + raise HTTPException(status_code=404, detail="Prediction results not found") + + uri = prediction_batch.file_path + if uri.startswith("s3://"): + try: + signed_url = artifact_store.presign_get(uri, expires_seconds=300) + except Exception as exc: + raise HTTPException(status_code=404, detail="Prediction artifact is unavailable") from exc + return RedirectResponse(url=signed_url, status_code=307) + + if not artifact_store.exists(uri): + raise HTTPException(status_code=404, detail="Prediction artifact has been deleted or moved") + return FileResponse( - path=file_path, + path=uri, filename=f"predictions_{prediction_id}.csv", - media_type="text/csv" + media_type="text/csv", ) diff --git a/Backend/app/api/session.py b/Backend/app/api/session.py new file mode 100644 index 0000000..134cdbc --- /dev/null +++ b/Backend/app/api/session.py @@ -0,0 +1,118 @@ +"""Anonymous temporary-session endpoints for guest-first NoCodeML.""" +from __future__ import annotations + +from typing import Annotated + +from fastapi import APIRouter, Depends, Form, Header, HTTPException, Response, status + +from app.core.config import settings +from app.services.session_manager import ( + InvalidSessionToken, + SessionExpired, + SessionNotFound, + session_manager, +) + + +router = APIRouter() +SESSION_HEADER = "X-NoCodeML-Session" + + +def _session_token( + token: Annotated[str | None, Header(alias=SESSION_HEADER)] = None, +) -> str: + if not token: + raise HTTPException( + status_code=status.HTTP_428_PRECONDITION_REQUIRED, + detail={ + "code": "SESSION_REQUIRED", + "message": "Start a temporary NoCodeML session before using this endpoint.", + }, + ) + return token + + +SessionToken = Annotated[str, Depends(_session_token)] + + +def _session_http_error(exc: Exception) -> HTTPException: + if isinstance(exc, InvalidSessionToken): + return HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={"code": "SESSION_INVALID", "message": "The temporary session token is invalid."}, + ) + if isinstance(exc, (SessionExpired, SessionNotFound)): + return HTTPException( + status_code=status.HTTP_410_GONE, + detail={ + "code": "SESSION_EXPIRED", + "message": "This temporary session has ended. Start a new session to continue.", + }, + ) + return HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail={"code": "SESSION_ERROR", "message": "The temporary workspace is unavailable."}, + ) + + +@router.post("", status_code=status.HTTP_201_CREATED) +def create_session(): + token, metadata = session_manager.create() + return { + "session_token": token, + "expires_at": metadata["expires_at"], + "ttl_seconds": settings.SESSION_TTL_MINUTES * 60, + "close_grace_seconds": settings.SESSION_CLOSE_GRACE_SECONDS, + "temporary": True, + "privacy": "Workspace files are temporary and are automatically removed after the session ends or expires.", + } + + +@router.get("") +def get_session(token: SessionToken): + try: + metadata = session_manager.touch(token) + except (InvalidSessionToken, SessionExpired, SessionNotFound) as exc: + raise _session_http_error(exc) from exc + + return { + "status": "active", + "expires_at": metadata["expires_at"], + "ttl_seconds": settings.SESSION_TTL_MINUTES * 60, + "temporary": True, + } + + +@router.post("/heartbeat") +def heartbeat_session(token: SessionToken): + try: + metadata = session_manager.touch(token) + except (InvalidSessionToken, SessionExpired, SessionNotFound) as exc: + raise _session_http_error(exc) from exc + return {"status": "active", "expires_at": metadata["expires_at"]} + + +@router.delete("", status_code=status.HTTP_204_NO_CONTENT) +def clear_session(token: SessionToken): + try: + session_manager.delete(token) + except InvalidSessionToken as exc: + raise _session_http_error(exc) from exc + return Response(status_code=status.HTTP_204_NO_CONTENT) + + +@router.post("/end", status_code=status.HTTP_202_ACCEPTED) +def mark_session_closing(session_token: Annotated[str, Form(min_length=32, max_length=128)]): + """Best-effort browser close signal using a CORS-safe form beacon. + + Deletion is delayed by a short grace period so normal page reloads can + heartbeat and keep the workspace alive. + """ + try: + session_manager.mark_closing(session_token) + except InvalidSessionToken as exc: + raise _session_http_error(exc) from exc + return { + "status": "cleanup_scheduled", + "delete_after_seconds": settings.SESSION_CLOSE_GRACE_SECONDS, + } diff --git a/Backend/app/api/workspace.py b/Backend/app/api/workspace.py new file mode 100644 index 0000000..c427a32 --- /dev/null +++ b/Backend/app/api/workspace.py @@ -0,0 +1,195 @@ +"""Guest-first temporary workspace endpoints.""" +from __future__ import annotations + +from typing import Annotated, Any, Literal + +from fastapi import APIRouter, File, Form, Query, UploadFile, status +from fastapi.responses import FileResponse +from pydantic import BaseModel, Field + +from app.api.session import SessionToken +from app.schemas.eda import EDAResponse, PlotRequest, PlotResponse +from app.services.workspace_dataset_service import ( + create_workspace_dataset, + delete_workspace_dataset, + get_workspace_dataset, + list_workspace_datasets, + preview_workspace_dataset, + update_workspace_dataset, +) +from app.services.workspace_eda_service import ( + generate_workspace_plot_data, + get_workspace_eda_summary, +) +from app.services.workspace_export_service import ( + build_session_bundle, + export_eda, + export_training, +) +from app.services.workspace_prediction_service import ( + list_predictions, + predict_batch, + predict_single, + prediction_download, +) +from app.services.workspace_training_service import workspace_training_runner + + +router = APIRouter() + + +class WorkspaceDatasetUpdate(BaseModel): + name: str = Field(min_length=1, max_length=200) + description: str | None = Field(default=None, max_length=1000) + + +class WorkspaceTrainingModel(BaseModel): + model_type: str = Field(min_length=1, max_length=100) + hyperparameters: dict[str, Any] = Field(default_factory=dict) + enable_optimization: bool = False + + +class WorkspaceTrainingRequest(BaseModel): + dataset_id: str = Field(min_length=1, max_length=100) + target_column: str = Field(min_length=1, max_length=300) + task_type: Literal["classification", "regression"] + selected_features: list[str] | None = None + models: list[WorkspaceTrainingModel] = Field(min_length=1, max_length=8) + test_size: float = Field(default=0.2, ge=0.1, le=0.4) + random_state: int = Field(default=42, ge=0, le=2_147_483_647) + cv_folds: int = Field(default=3, ge=2, le=5) + scaling: bool = True + + +class WorkspaceSinglePredictionRequest(BaseModel): + features: dict[str, Any] + + +@router.post("/datasets", status_code=status.HTTP_201_CREATED) +async def upload_dataset( + token: SessionToken, + file: Annotated[UploadFile, File(...)], + name: Annotated[str | None, Form()] = None, + description: Annotated[str | None, Form()] = None, +): + dataset = await create_workspace_dataset(token, file, name, description) + return {"dataset": dataset} + + +@router.get("/datasets") +def list_datasets(token: SessionToken): + datasets = list_workspace_datasets(token) + return {"datasets": datasets, "total": len(datasets), "temporary": True} + + +@router.get("/datasets/{dataset_id}") +def get_dataset(dataset_id: str, token: SessionToken): + return {"dataset": get_workspace_dataset(token, dataset_id)} + + +@router.put("/datasets/{dataset_id}") +def update_dataset(dataset_id: str, payload: WorkspaceDatasetUpdate, token: SessionToken): + return { + "dataset": update_workspace_dataset( + token, + dataset_id, + payload.name, + payload.description, + ) + } + + +@router.get("/datasets/{dataset_id}/preview") +def preview_dataset( + dataset_id: str, + token: SessionToken, + rows: Annotated[int, Query(ge=1, le=50)] = 10, +): + return preview_workspace_dataset(token, dataset_id, rows) + + +@router.get("/datasets/{dataset_id}/eda", response_model=EDAResponse) +async def dataset_eda(dataset_id: str, token: SessionToken): + return await get_workspace_eda_summary(token, dataset_id) + + +@router.post("/datasets/{dataset_id}/plot", response_model=PlotResponse) +async def dataset_plot(dataset_id: str, request: PlotRequest, token: SessionToken): + return await generate_workspace_plot_data( + token=token, + dataset_id=dataset_id, + plot_type=request.plot_type, + x_column=request.x_column, + y_column=request.y_column, + group_by=request.group_by, + ) + + +@router.get("/datasets/{dataset_id}/export/{kind}") +async def download_eda_export(dataset_id: str, kind: str, token: SessionToken): + file_path, download_name, media_type = await export_eda(token, dataset_id, kind) + return FileResponse(path=file_path, media_type=media_type, filename=download_name) + + +@router.delete("/datasets/{dataset_id}", status_code=status.HTTP_204_NO_CONTENT) +def delete_dataset(dataset_id: str, token: SessionToken): + delete_workspace_dataset(token, dataset_id) + return None + + +@router.post("/training/runs", status_code=status.HTTP_202_ACCEPTED) +def start_training(payload: WorkspaceTrainingRequest, token: SessionToken): + return workspace_training_runner.submit(token, payload.model_dump()) + + +@router.get("/training/runs") +def list_training_runs(token: SessionToken): + runs = workspace_training_runner.list(token) + return {"runs": runs, "total": len(runs), "temporary": True} + + +@router.get("/training/runs/{run_id}") +def get_training_run(run_id: str, token: SessionToken): + return workspace_training_runner.get(token, run_id) + + +@router.get("/training/runs/{run_id}/export/{kind}") +def download_training_export(run_id: str, kind: str, token: SessionToken): + file_path, download_name, media_type = export_training(token, run_id, kind) + return FileResponse(path=file_path, media_type=media_type, filename=download_name) + + +@router.post("/training/runs/{run_id}/predict") +def single_prediction(run_id: str, payload: WorkspaceSinglePredictionRequest, token: SessionToken): + return predict_single(token, run_id, payload.features) + + +@router.post("/training/runs/{run_id}/predict/batch", status_code=status.HTTP_201_CREATED) +async def batch_prediction( + run_id: str, + token: SessionToken, + file: Annotated[UploadFile, File(...)], +): + return await predict_batch(token, run_id, file) + + +@router.get("/predictions") +def prediction_history(token: SessionToken): + predictions = list_predictions(token) + return {"predictions": predictions, "total": len(predictions), "temporary": True} + + +@router.get("/predictions/{prediction_id}/download") +def download_prediction(prediction_id: str, token: SessionToken): + file_path, download_name = prediction_download(token, prediction_id) + return FileResponse( + path=file_path, + media_type="text/csv", + filename=download_name, + ) + + +@router.get("/export/session") +async def download_session(token: SessionToken): + file_path, download_name = await build_session_bundle(token) + return FileResponse(path=file_path, media_type="application/zip", filename=download_name) diff --git a/Backend/app/core/config.py b/Backend/app/core/config.py index aaf7a55..78f328e 100644 --- a/Backend/app/core/config.py +++ b/Backend/app/core/config.py @@ -1,41 +1,139 @@ +from pathlib import Path from typing import List -from pydantic_settings import BaseSettings + +from pydantic import model_validator +from pydantic_settings import BaseSettings, SettingsConfigDict + class Settings(BaseSettings): - """Application settings loaded from environment variables.""" + """NoCodeML runtime settings. + + V3 visitor workflows are intentionally database-free. Legacy database and + Celery settings remain optional only so archived modules can still be read + or exercised in development without becoming production dependencies. + """ + + model_config = SettingsConfigDict(env_file=".env", extra="ignore") + PROJECT_NAME: str = "NoCodeML API" + APP_VERSION: str = "3.0.0" API_V1_STR: str = "/api/v1" - - # Database - DATABASE_URL: str = "sqlite+aiosqlite:///./nocodeml.db" - - # Redis (for Celery) + ENVIRONMENT: str = "development" + + # Optional legacy compatibility. The public V3 runtime does not connect to + # or write visitor data into this database. + DATABASE_URL: str = "sqlite+aiosqlite:////tmp/nocodeml-legacy.db" + DB_SCHEMA: str = "nocodeml" + + # Temporary guest workspaces. Raw session tokens are never folder names. + SESSION_ROOT_DIR: str = "/tmp/nocodeml-sessions" + SESSION_TTL_MINUTES: int = 60 + SESSION_CLEANUP_INTERVAL_SECONDS: int = 300 + SESSION_CLOSE_GRACE_SECONDS: int = 30 + + # Bounded in-process ML capacity for the single-instance guest backend. + WORKSPACE_TRAINING_WORKERS: int = 1 + WORKSPACE_MAX_MODELS_PER_RUN: int = 8 + + # Legacy/local artifact settings retained for archived services only. + DATASETS_DIR: str = "/tmp/nocodeml-legacy/datasets" + MODELS_DIR: str = "/tmp/nocodeml-legacy/models" + PREDICTIONS_DIR: str = "/tmp/nocodeml-legacy/predictions" + ARTIFACT_CACHE_DIR: str = "/tmp/nocodeml-artifacts" + ARTIFACT_STORAGE_BACKEND: str = "local" + S3_ENDPOINT_URL: str = "" + S3_ACCESS_KEY_ID: str = "" + S3_SECRET_ACCESS_KEY: str = "" + S3_BUCKET_NAME: str = "" + S3_REGION: str = "auto" + S3_ADDRESSING_STYLE: str = "path" + CELERY_BROKER_URL: str = "memory://" CELERY_RESULT_BACKEND: str = "cache+memory://" - - # JWT Authentication - SECRET_KEY: str = "local-development-key-change-before-deployment" + + # Legacy auth modules are not mounted in the V3 public API. + SECRET_KEY: str = "legacy-auth-disabled-in-v3-guest-runtime" ALGORITHM: str = "HS256" ACCESS_TOKEN_EXPIRE_MINUTES: int = 60 - # CORS - comma-separated list of allowed origins + # Data Science Assistant. Key remains server-side only. + GEMINI_API_KEY: str = "" + GEMINI_MODEL: str = "gemini-3.7-flash" + BACKEND_CORS_ORIGINS: str = ( - "https://www.nocodeml.cloud," - "https://nocodeml.cloud," "http://localhost:5173," "http://127.0.0.1:5173," "http://localhost:5174," - "http://127.0.0.1:5174," - "http://localhost:3000," - "http://localhost:8080" + "http://127.0.0.1:5174" ) @property def cors_origins(self) -> List[str]: - return [o.strip() for o in self.BACKEND_CORS_ORIGINS.split(",") if o.strip()] + return [origin.strip().rstrip("/") for origin in self.BACKEND_CORS_ORIGINS.split(",") if origin.strip()] + + @property + def is_postgres(self) -> bool: + return self.DATABASE_URL.startswith("postgresql") + + @property + def database_connect_args(self) -> dict: + if not self.is_postgres: + return {} + return {"options": f"-csearch_path={self.DB_SCHEMA}"} + + @property + def storage_paths(self) -> tuple[Path, Path, Path]: + return tuple(Path(path).expanduser() for path in (self.DATASETS_DIR, self.MODELS_DIR, self.PREDICTIONS_DIR)) + + @property + def session_root(self) -> Path: + return Path(self.SESSION_ROOT_DIR).expanduser() + + @property + def uses_object_storage(self) -> bool: + return self.ARTIFACT_STORAGE_BACKEND.lower() == "s3" + + @model_validator(mode="after") + def validate_runtime_safety(self): + for field_name in ("SESSION_ROOT_DIR", "DATASETS_DIR", "MODELS_DIR", "PREDICTIONS_DIR", "ARTIFACT_CACHE_DIR"): + value = getattr(self, field_name).strip() + if not value: + raise ValueError(f"{field_name} cannot be empty") + setattr(self, field_name, value) + + if not 5 <= self.SESSION_TTL_MINUTES <= 24 * 60: + raise ValueError("SESSION_TTL_MINUTES must be between 5 and 1440") + if not 10 <= self.SESSION_CLEANUP_INTERVAL_SECONDS <= 3600: + raise ValueError("SESSION_CLEANUP_INTERVAL_SECONDS must be between 10 and 3600") + if not 5 <= self.SESSION_CLOSE_GRACE_SECONDS <= 300: + raise ValueError("SESSION_CLOSE_GRACE_SECONDS must be between 5 and 300") + if not 1 <= self.WORKSPACE_TRAINING_WORKERS <= 4: + raise ValueError("WORKSPACE_TRAINING_WORKERS must be between 1 and 4") + if not 1 <= self.WORKSPACE_MAX_MODELS_PER_RUN <= 8: + raise ValueError("WORKSPACE_MAX_MODELS_PER_RUN must be between 1 and 8") + + backend = self.ARTIFACT_STORAGE_BACKEND.strip().lower() + if backend not in {"local", "s3"}: + raise ValueError("ARTIFACT_STORAGE_BACKEND must be 'local' or 's3'") + self.ARTIFACT_STORAGE_BACKEND = backend + + style = self.S3_ADDRESSING_STYLE.strip().lower() + if style not in {"path", "virtual", "auto"}: + raise ValueError("S3_ADDRESSING_STYLE must be path, virtual, or auto") + self.S3_ADDRESSING_STYLE = style + + if backend == "s3": + required = { + "S3_ENDPOINT_URL": self.S3_ENDPOINT_URL, + "S3_ACCESS_KEY_ID": self.S3_ACCESS_KEY_ID, + "S3_SECRET_ACCESS_KEY": self.S3_SECRET_ACCESS_KEY, + "S3_BUCKET_NAME": self.S3_BUCKET_NAME, + } + missing = [name for name, value in required.items() if not value.strip()] + if missing: + raise ValueError(f"Missing S3 artifact settings: {', '.join(missing)}") + + return self - class Config: - env_file = ".env" - extra = "ignore" settings = Settings() diff --git a/Backend/app/db/session.py b/Backend/app/db/session.py index 046950a..5cc7f1d 100644 --- a/Backend/app/db/session.py +++ b/Backend/app/db/session.py @@ -1,24 +1,28 @@ -from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker, AsyncSession -from app.core.config import settings from typing import AsyncGenerator -# psycopg3 async driver - URL should be postgresql+psycopg://... -# The create_async_engine will use psycopg in async mode automatically +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine + +from app.core.config import settings + + async_engine = create_async_engine( settings.DATABASE_URL, + connect_args=settings.database_connect_args, pool_pre_ping=True, echo=False, - future=True # Use SQLAlchemy 2.0 style + future=True, ) + AsyncSessionLocal = async_sessionmaker( - autocommit=False, - autoflush=False, + autocommit=False, + autoflush=False, bind=async_engine, class_=AsyncSession, - expire_on_commit=False + expire_on_commit=False, ) + async def get_db() -> AsyncGenerator[AsyncSession, None]: """Dependency for getting async database sessions.""" async with AsyncSessionLocal() as session: - yield session \ No newline at end of file + yield session diff --git a/Backend/app/db/sync_session.py b/Backend/app/db/sync_session.py index aad6d5a..eaeeea4 100644 --- a/Backend/app/db/sync_session.py +++ b/Backend/app/db/sync_session.py @@ -1,25 +1,33 @@ from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker + from app.core.config import settings + sync_database_url = settings.DATABASE_URL.replace( "sqlite+aiosqlite://", "sqlite://", 1 ) -# psycopg3 supports both sync and async with the same driver -# For sync, we use create_engine with postgresql+psycopg URL (not create_async_engine) -# The psycopg[binary] package includes both sync and async support -sync_engine = create_engine( - sync_database_url, - pool_pre_ping=True, - pool_size=5, - max_overflow=10, - pool_recycle=3600, - echo=False, - future=True # Use SQLAlchemy 2.0 style -) +sync_engine_kwargs = { + "pool_pre_ping": True, + "echo": False, + "future": True, +} + +if sync_database_url.startswith("postgresql"): + sync_engine_kwargs.update( + { + "connect_args": settings.database_connect_args, + "pool_size": 5, + "max_overflow": 10, + "pool_recycle": 3600, + } + ) + +sync_engine = create_engine(sync_database_url, **sync_engine_kwargs) SyncSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=sync_engine) + def get_sync_db(): db = SyncSessionLocal() try: diff --git a/Backend/app/main.py b/Backend/app/main.py index 16c822f..6a976e8 100644 --- a/Backend/app/main.py +++ b/Backend/app/main.py @@ -1,64 +1,120 @@ -"""Main FastAPI application entry point.""" -from fastapi import FastAPI +"""Main FastAPI application entry point for guest-first NoCodeML.""" +import asyncio +import shutil +import tempfile +from contextlib import asynccontextmanager, suppress + +from fastapi import FastAPI, Response, status from fastapi.middleware.cors import CORSMiddleware -from contextlib import asynccontextmanager -from app.core.config import settings + from app.api import api_router -from app.db.session import async_engine -from app.models import Base +from app.core.config import settings from app.core.model_cache import initialize_model_cache +from app.services.session_manager import session_manager + + +async def _session_cleanup_loop() -> None: + while True: + await asyncio.sleep(settings.SESSION_CLEANUP_INTERVAL_SECONDS) + try: + removed = await asyncio.to_thread(session_manager.cleanup_expired) + if removed: + print(f"Temporary session cleanup removed {removed} workspace(s)") + except Exception as exc: + print(f"Temporary session cleanup warning: {type(exc).__name__}") @asynccontextmanager async def lifespan(app: FastAPI): - """Manage application startup and shutdown events.""" - if settings.DATABASE_URL.startswith("sqlite+"): - # Local mode: create the SQLite schema automatically. PostgreSQL - # deployments continue to use Alembic migrations. - async with async_engine.begin() as conn: - await conn.run_sync(Base.metadata.create_all) - - # Note: Table creation is handled by Alembic migrations - # Run: docker exec fastapi_app alembic upgrade head - print("Application started - database is ready") - - # Initialize model cache on startup initialize_model_cache() - print("Model cache initialized") - - yield - - await async_engine.dispose() - print("Database connections closed") + session_manager.ensure_root() + reset_leases = await asyncio.to_thread(session_manager.reset_stale_job_leases) + if reset_leases: + print(f"Recovered {reset_leases} interrupted temporary training session(s)") + await asyncio.to_thread(session_manager.cleanup_expired) + cleanup_task = asyncio.create_task(_session_cleanup_loop(), name="nocodeml-session-cleanup") + print(f"NoCodeML {settings.APP_VERSION} started in temporary guest mode") + try: + yield + finally: + cleanup_task.cancel() + with suppress(asyncio.CancelledError): + await cleanup_task + print("NoCodeML shutdown complete") app = FastAPI( title=settings.PROJECT_NAME, lifespan=lifespan, - description="NoCodeML API - Build ML models without code, powered by custom JWT authentication", - version="2.0.0" + description="NoCodeML V3 API - temporary, account-free visual machine learning", + version=settings.APP_VERSION, ) app.add_middleware( CORSMiddleware, allow_origins=settings.cors_origins, - allow_credentials=True, - allow_methods=["*"], - allow_headers=["*"], + allow_credentials=False, + allow_methods=["GET", "POST", "PUT", "DELETE", "OPTIONS"], + allow_headers=["Content-Type", "Accept", "X-NoCodeML-Session"], + expose_headers=["Content-Disposition"], ) + @app.get("/") def read_root(): - """API welcome message.""" return { "message": f"Welcome to {settings.PROJECT_NAME}", - "version": "2.0.0", - "docs": "/docs" + "version": settings.APP_VERSION, + "mode": "temporary-guest", + "persistence": "disabled-for-visitor-workspaces", + "docs": "/docs", } + @app.get("/health") def health_check(): - """Health check endpoint.""" - return {"status": "healthy", "service": "NoCodeML API"} + return { + "status": "healthy", + "service": "NoCodeML API", + "version": settings.APP_VERSION, + "mode": "temporary-guest", + } + + +@app.get("/ready") +def readiness_check(response: Response): + checks: dict[str, dict[str, object]] = {} + try: + root = session_manager.ensure_root() + with tempfile.NamedTemporaryFile(prefix=".ready-", dir=root): + pass + usage = shutil.disk_usage(root) + checks["temporary_workspace"] = { + "status": "ready", + "free_mb": round(usage.free / 1024 / 1024), + } + except Exception: + checks["temporary_workspace"] = {"status": "unavailable"} + + checks["training"] = { + "status": "ready", + "backend": "bounded-in-process", + "workers": settings.WORKSPACE_TRAINING_WORKERS, + "max_models_per_run": settings.WORKSPACE_MAX_MODELS_PER_RUN, + } + + ready = all(check["status"] != "unavailable" for check in checks.values()) + if not ready: + response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE + + return { + "status": "ready" if ready else "not_ready", + "service": "NoCodeML API", + "version": settings.APP_VERSION, + "environment": settings.ENVIRONMENT, + "mode": "temporary-guest", + "checks": checks, + } + app.include_router(api_router, prefix=settings.API_V1_STR) diff --git a/Backend/app/schemas/__init__.py b/Backend/app/schemas/__init__.py index 632ffe9..955626a 100644 --- a/Backend/app/schemas/__init__.py +++ b/Backend/app/schemas/__init__.py @@ -1,25 +1,36 @@ """Pydantic schemas for request/response validation.""" from datetime import datetime from typing import Optional + from pydantic import BaseModel, EmailStr, field_validator class UserCreate(BaseModel): """Schema for user registration request.""" + email: EmailStr password: str - - @field_validator('password') + + @field_validator("email") + @classmethod + def normalize_email(cls, value: EmailStr) -> str: + return str(value).strip().lower() + + @field_validator("password") @classmethod - def validate_password(cls, v: str) -> str: - """Validate password meets minimum requirements.""" - if len(v) < 8: - raise ValueError('Password must be at least 8 characters long') - return v + def validate_password(cls, value: str) -> str: + if len(value) < 8: + raise ValueError("Password must be at least 8 characters long") + # bcrypt operates on at most 72 bytes. Reject oversized inputs instead of + # silently hashing a truncated password. + if len(value.encode("utf-8")) > 72: + raise ValueError("Password must be at most 72 UTF-8 bytes") + return value class UserResponse(BaseModel): """Schema for user data in responses.""" + id: int email: str is_active: bool @@ -27,62 +38,66 @@ class UserResponse(BaseModel): is_verified: bool created_at: datetime updated_at: Optional[datetime] = None - + class Config: from_attributes = True class LoginRequest(BaseModel): """Schema for login request.""" + email: EmailStr password: str + @field_validator("email") + @classmethod + def normalize_email(cls, value: EmailStr) -> str: + return str(value).strip().lower() + class Token(BaseModel): """Schema for token response.""" + access_token: str token_type: str = "bearer" from app.schemas.dataset import ( + ColumnInfo, DatasetCreate, - DatasetUpdate, - DatasetResponse, DatasetListResponse, DatasetPreviewResponse, - ColumnInfo + DatasetResponse, + DatasetUpdate, ) - from app.schemas.experiment import ( ExperimentConfig, ExperimentCreate, - ExperimentUpdate, + ExperimentListResponse, ExperimentResponse, - ExperimentListResponse + ExperimentUpdate, ) - from app.schemas.model_config import ( - PreprocessingConfig, - TrainingConfig, ModelConfigRequest, ModelConfigResponse, ModelSelectionRequest, - ModelSelectionResponse + ModelSelectionResponse, + PreprocessingConfig, + TrainingConfig, ) - from app.schemas.training import ( + ConfusionMatrix, + ExperimentTrainingStatus, + FeatureImportance, + JobStatusResponse, + StartTrainingRequest, + StartTrainingResponse, TrainingJobCreate, TrainingJobResponse, - TrainingProgress, + TrainingLogResponse, TrainingMetrics, - FeatureImportance, - ConfusionMatrix, + TrainingProgress, TrainingResultResponse, - TrainingLogResponse, - StartTrainingRequest, - StartTrainingResponse, - JobStatusResponse, - ExperimentTrainingStatus ) __all__ = [ @@ -118,5 +133,5 @@ class Token(BaseModel): "StartTrainingRequest", "StartTrainingResponse", "JobStatusResponse", - "ExperimentTrainingStatus" + "ExperimentTrainingStatus", ] diff --git a/Backend/app/services/artifact_store.py b/Backend/app/services/artifact_store.py new file mode 100644 index 0000000..e5c3323 --- /dev/null +++ b/Backend/app/services/artifact_store.py @@ -0,0 +1,159 @@ +"""Artifact storage abstraction for NoCodeML-owned files. + +Development uses the local filesystem. Production may use a private S3-compatible +bucket so the FastAPI and Celery services can share datasets, trained models and +prediction exports without sharing a filesystem or another application's storage. +""" +from __future__ import annotations + +import shutil +import tempfile +from contextlib import contextmanager +from pathlib import Path +from typing import Iterator, Optional +from urllib.parse import urlparse + +from app.core.config import settings + + +class ArtifactStore: + def __init__(self) -> None: + self.backend = settings.ARTIFACT_STORAGE_BACKEND + self.bucket = settings.S3_BUCKET_NAME + self.cache_dir = Path(settings.ARTIFACT_CACHE_DIR).expanduser() + self.cache_dir.mkdir(parents=True, exist_ok=True) + self._client = None + + @property + def is_remote(self) -> bool: + return self.backend == "s3" + + def _s3_client(self): + if not self.is_remote: + raise RuntimeError("S3 client requested while local artifact storage is active") + if self._client is None: + import boto3 + from botocore.config import Config + + region = settings.S3_REGION.strip() + self._client = boto3.client( + "s3", + endpoint_url=settings.S3_ENDPOINT_URL.rstrip("/"), + aws_access_key_id=settings.S3_ACCESS_KEY_ID, + aws_secret_access_key=settings.S3_SECRET_ACCESS_KEY, + region_name=None if region in {"", "auto"} else region, + config=Config( + signature_version="s3v4", + s3={"addressing_style": settings.S3_ADDRESSING_STYLE}, + retries={"max_attempts": 3, "mode": "standard"}, + ), + ) + return self._client + + @staticmethod + def _clean_key(key: str) -> str: + normalized = key.replace("\\", "/").lstrip("/") + parts = [part for part in normalized.split("/") if part not in {"", "."}] + if not parts or any(part == ".." for part in parts): + raise ValueError("Invalid artifact key") + return "/".join(parts) + + def _parse_s3_uri(self, uri: str) -> tuple[str, str]: + parsed = urlparse(uri) + if parsed.scheme != "s3" or not parsed.netloc or not parsed.path: + raise ValueError("Invalid S3 artifact URI") + return parsed.netloc, self._clean_key(parsed.path) + + def put_file(self, local_path: str | Path, key: str, content_type: Optional[str] = None) -> str: + """Persist a local file and return the canonical artifact URI/path.""" + source = Path(local_path) + if not source.is_file(): + raise FileNotFoundError(f"Artifact source not found: {source}") + + if not self.is_remote: + return str(source) + + object_key = self._clean_key(key) + extra_args = {"ContentType": content_type} if content_type else None + kwargs = {"ExtraArgs": extra_args} if extra_args else {} + self._s3_client().upload_file(str(source), self.bucket, object_key, **kwargs) + return f"s3://{self.bucket}/{object_key}" + + def delete(self, uri: str) -> bool: + """Delete a NoCodeML artifact. Missing artifacts are treated idempotently.""" + if uri.startswith("s3://"): + bucket, key = self._parse_s3_uri(uri) + if bucket != self.bucket: + raise ValueError("Refusing to delete an artifact outside the configured NoCodeML bucket") + self._s3_client().delete_object(Bucket=bucket, Key=key) + return True + + path = Path(uri) + if path.exists() and path.is_file(): + path.unlink() + return True + return False + + def exists(self, uri: str) -> bool: + if uri.startswith("s3://"): + bucket, key = self._parse_s3_uri(uri) + if bucket != self.bucket: + return False + try: + self._s3_client().head_object(Bucket=bucket, Key=key) + return True + except Exception: + return False + return Path(uri).is_file() + + @contextmanager + def materialize(self, uri: str) -> Iterator[Path]: + """Yield a readable local path for a local or remote artifact.""" + if not uri.startswith("s3://"): + path = Path(uri) + if not path.is_file(): + raise FileNotFoundError(f"Artifact not found: {path}") + yield path + return + + bucket, key = self._parse_s3_uri(uri) + if bucket != self.bucket: + raise ValueError("Refusing to read an artifact outside the configured NoCodeML bucket") + + suffix = Path(key).suffix + with tempfile.NamedTemporaryFile( + dir=self.cache_dir, + suffix=suffix, + prefix="artifact-", + delete=False, + ) as handle: + temp_path = Path(handle.name) + + try: + self._s3_client().download_file(bucket, key, str(temp_path)) + yield temp_path + finally: + temp_path.unlink(missing_ok=True) + + def presign_get(self, uri: str, expires_seconds: int = 300) -> str: + if not uri.startswith("s3://"): + raise ValueError("Presigned URLs are only available for S3 artifacts") + bucket, key = self._parse_s3_uri(uri) + if bucket != self.bucket: + raise ValueError("Refusing to sign an artifact outside the configured NoCodeML bucket") + return self._s3_client().generate_presigned_url( + "get_object", + Params={"Bucket": bucket, "Key": key}, + ExpiresIn=max(60, min(expires_seconds, 900)), + ) + + def copy_local(self, source: str | Path, destination: str | Path) -> str: + """Copy a local artifact when callers need an explicit local destination.""" + src = Path(source) + dst = Path(destination) + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src, dst) + return str(dst) + + +artifact_store = ArtifactStore() diff --git a/Backend/app/services/dataset_service.py b/Backend/app/services/dataset_service.py index 64d7625..3f45c19 100644 --- a/Backend/app/services/dataset_service.py +++ b/Backend/app/services/dataset_service.py @@ -1,21 +1,26 @@ """Dataset service layer for business logic.""" -import os +from __future__ import annotations + +import asyncio import uuid from pathlib import Path -from typing import List, Optional, Dict, Any -from fastapi import UploadFile, HTTPException, status -from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy import select, func +from typing import Any, Dict, List, Optional + import pandas as pd +from fastapi import HTTPException, UploadFile, status +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import AsyncSession -from app.models.dataset import Dataset from app.core.config import settings +from app.models.dataset import Dataset +from app.services.artifact_store import artifact_store -UPLOAD_DIR = Path("./datasets") +UPLOAD_DIR = Path(settings.DATASETS_DIR).expanduser() MAX_FILE_SIZE = 100 * 1024 * 1024 -ALLOWED_EXTENSIONS = {'.csv', '.xlsx', '.xls', '.parquet'} +ALLOWED_EXTENSIONS = {".csv", ".xlsx", ".xls", ".parquet"} MAX_PREVIEW_ROWS = 50 +UPLOAD_CHUNK_SIZE = 1024 * 1024 async def create_dataset( @@ -23,67 +28,96 @@ async def create_dataset( file: UploadFile, name: str, description: Optional[str], - user_id: int + user_id: int, ) -> Dataset: - """Create a new dataset from uploaded file.""" - file_ext = Path(file.filename).suffix.lower() + """Create a new dataset from a validated uploaded file.""" + clean_name = name.strip() + if not clean_name: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Dataset name cannot be empty") + + original_filename = Path(file.filename or "dataset").name + file_ext = Path(original_filename).suffix.lower() if file_ext not in ALLOWED_EXTENSIONS: + allowed = ", ".join(sorted(ALLOWED_EXTENSIONS)) raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, - detail=f"File type '{file_ext}' not supported. Allowed types: {', '.join(ALLOWED_EXTENSIONS)}" + detail=f"Unsupported file type. Allowed types: {allowed}", ) - + dataset_id = uuid.uuid4() user_dir = UPLOAD_DIR / str(user_id) user_dir.mkdir(parents=True, exist_ok=True) - - storage_filename = f"{dataset_id}_{file.filename}" - storage_path = user_dir / storage_filename - + + if artifact_store.is_remote: + staging_dir = UPLOAD_DIR / ".staging" / str(user_id) + staging_dir.mkdir(parents=True, exist_ok=True) + local_path = staging_dir / f"{dataset_id}{file_ext}" + else: + local_path = user_dir / f"{dataset_id}{file_ext}" + + artifact_uri: Optional[str] = None + try: - file_size = await save_upload_file(file, storage_path) - - if file_size > MAX_FILE_SIZE: - storage_path.unlink(missing_ok=True) - raise HTTPException( - status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, - detail=f"File too large. Maximum size is {MAX_FILE_SIZE / (1024*1024):.0f}MB" + file_size = await save_upload_file(file, local_path, MAX_FILE_SIZE) + metadata = await asyncio.to_thread(extract_file_metadata, str(local_path)) + + if artifact_store.is_remote: + artifact_uri = await asyncio.to_thread( + artifact_store.put_file, + local_path, + f"datasets/{user_id}/{dataset_id}{file_ext}", + file.content_type, ) - except Exception as e: - storage_path.unlink(missing_ok=True) - if isinstance(e, HTTPException): - raise e - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Failed to save file: {str(e)}" - ) - - try: - metadata = await extract_file_metadata(str(storage_path)) - except Exception as e: - storage_path.unlink(missing_ok=True) + local_path.unlink(missing_ok=True) + else: + artifact_uri = str(local_path) + except HTTPException: + local_path.unlink(missing_ok=True) + if artifact_uri: + await _delete_artifact_quietly(artifact_uri) + raise + except (ValueError, pd.errors.ParserError, UnicodeDecodeError) as exc: + local_path.unlink(missing_ok=True) + if artifact_uri: + await _delete_artifact_quietly(artifact_uri) raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Failed to read file: {str(e)}. Please ensure the file is valid and not corrupted." - ) - + detail="The uploaded dataset could not be parsed. Check that the file is valid and not corrupted.", + ) from exc + except Exception as exc: + local_path.unlink(missing_ok=True) + if artifact_uri: + await _delete_artifact_quietly(artifact_uri) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="The dataset could not be processed.", + ) from exc + finally: + await file.close() + dataset = Dataset( id=dataset_id, user_id=user_id, - name=name, - description=description, - storage_path=str(storage_path), - file_name=file.filename, + name=clean_name, + description=description.strip() if description else None, + storage_path=artifact_uri, + file_name=original_filename, file_size_bytes=file_size, - row_count=metadata['row_count'], - column_count=metadata['column_count'], - column_info=metadata['column_info'] + row_count=metadata["row_count"], + column_count=metadata["column_count"], + column_info=metadata["column_info"], ) - + db.add(dataset) - await db.commit() - await db.refresh(dataset) - + try: + await db.commit() + await db.refresh(dataset) + except Exception: + await db.rollback() + if artifact_uri: + await _delete_artifact_quietly(artifact_uri) + raise + return dataset @@ -91,13 +125,15 @@ async def get_user_datasets( db: AsyncSession, user_id: int, skip: int = 0, - limit: int = 100 + limit: int = 100, ) -> tuple[List[Dataset], int]: - """Fetch all datasets for a user with pagination.""" + skip = max(0, skip) + limit = max(1, min(limit, 100)) + count_query = select(func.count()).select_from(Dataset).where(Dataset.user_id == user_id) total_result = await db.execute(count_query) total = total_result.scalar() or 0 - + query = ( select(Dataset) .where(Dataset.user_id == user_id) @@ -106,21 +142,15 @@ async def get_user_datasets( .limit(limit) ) result = await db.execute(query) - datasets = result.scalars().all() - - return list(datasets), total + return list(result.scalars().all()), total async def get_dataset_by_id( db: AsyncSession, dataset_id: uuid.UUID, - user_id: int + user_id: int, ) -> Optional[Dataset]: - """Fetch a single dataset and verify ownership.""" - query = select(Dataset).where( - Dataset.id == dataset_id, - Dataset.user_id == user_id - ) + query = select(Dataset).where(Dataset.id == dataset_id, Dataset.user_id == user_id) result = await db.execute(query) return result.scalar_one_or_none() @@ -130,81 +160,65 @@ async def update_dataset( dataset_id: uuid.UUID, user_id: int, name: str, - description: Optional[str] + description: Optional[str], ) -> Optional[Dataset]: - """Update dataset name and description.""" dataset = await get_dataset_by_id(db, dataset_id, user_id) if not dataset: return None - - dataset.name = name - dataset.description = description - + + clean_name = name.strip() + if not clean_name: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Dataset name cannot be empty") + + dataset.name = clean_name + dataset.description = description.strip() if description else None await db.commit() await db.refresh(dataset) - return dataset async def check_dataset_dependencies( db: AsyncSession, dataset_id: uuid.UUID, - user_id: int + user_id: int, ) -> None: - """ - Check if dataset is used by any experiments. - - Args: - db: Database session - dataset_id: ID of the dataset - user_id: ID of the user - - Raises: - HTTPException: 409 if experiments depend on this dataset - """ from app.models.experiment import Experiment - - # Query for experiments using this dataset + query = select(Experiment).where( Experiment.dataset_id == dataset_id, - Experiment.user_id == user_id + Experiment.user_id == user_id, ) result = await db.execute(query) experiments = result.scalars().all() - + if experiments: - experiment_names = [exp.name for exp in experiments] raise HTTPException( status_code=status.HTTP_409_CONFLICT, - detail=f"Cannot delete dataset. It is used by {len(experiments)} experiment(s): {', '.join(experiment_names)}" + detail={ + "message": f"Cannot delete dataset while {len(experiments)} experiment(s) still use it.", + "dependencies": [ + {"id": str(experiment.id), "name": experiment.name} + for experiment in experiments + ], + }, ) async def delete_dataset( db: AsyncSession, dataset_id: uuid.UUID, - user_id: int + user_id: int, ) -> bool: - """Delete a dataset (file and database record).""" - # Fetch and verify ownership dataset = await get_dataset_by_id(db, dataset_id, user_id) if not dataset: return False - - # Check for dependencies (experiments using this dataset) + await check_dataset_dependencies(db, dataset_id, user_id) - - # Delete physical file - try: - delete_file(dataset.storage_path) - except Exception as e: - # Log the error but continue with database deletion - print(f"Warning: Failed to delete file {dataset.storage_path}: {str(e)}") - - # Delete database record + + artifact_uri = dataset.storage_path await db.delete(dataset) await db.commit() - + await _delete_artifact_quietly(artifact_uri) return True @@ -212,98 +226,95 @@ async def get_dataset_preview( db: AsyncSession, dataset_id: uuid.UUID, user_id: int, - rows: int = 10 + rows: int = 10, ) -> Optional[Dict[str, Any]]: - """Get a preview of dataset contents.""" - # Fetch and verify ownership dataset = await get_dataset_by_id(db, dataset_id, user_id) if not dataset: return None - - # Limit preview rows - rows = min(rows, MAX_PREVIEW_ROWS) - - # Read file and extract preview + + rows = max(1, min(rows, MAX_PREVIEW_ROWS)) + try: - file_ext = Path(dataset.storage_path).suffix.lower() - - if file_ext == '.csv': - df = pd.read_csv(dataset.storage_path, nrows=rows) - elif file_ext in ['.xlsx', '.xls']: - df = pd.read_excel(dataset.storage_path, nrows=rows) - elif file_ext == '.parquet': - df = pd.read_parquet(dataset.storage_path) - df = df.head(rows) - else: - raise ValueError(f"Unsupported file type: {file_ext}") - - df_filled = df.where(pd.notna(df), None) - - return { - "columns": df.columns.tolist(), - "data": df_filled.values.tolist(), - "row_count": dataset.row_count, - "preview_rows": len(df) - } - except Exception as e: + return await asyncio.to_thread(read_dataset_preview, dataset.storage_path, dataset.row_count, rows) + except Exception as exc: raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Failed to read dataset preview: {str(e)}" - ) + detail="The dataset preview could not be generated.", + ) from exc -async def save_upload_file(file: UploadFile, destination: Path) -> int: - """Save uploaded file to filesystem.""" +async def save_upload_file(file: UploadFile, destination: Path, max_size: int) -> int: + """Stream an upload to disk and stop as soon as it exceeds the allowed size.""" file_size = 0 - - with open(destination, 'wb') as buffer: - while chunk := await file.read(8192): - buffer.write(chunk) + with destination.open("wb") as buffer: + while True: + chunk = await file.read(UPLOAD_CHUNK_SIZE) + if not chunk: + break file_size += len(chunk) - + if file_size > max_size: + raise HTTPException( + status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, + detail=f"File too large. Maximum size is {max_size // (1024 * 1024)} MB.", + ) + buffer.write(chunk) return file_size -async def extract_file_metadata(file_path: str) -> Dict[str, Any]: - """Extract metadata from a dataset file.""" - file_ext = Path(file_path).suffix.lower() - - # Read file based on extension - if file_ext == '.csv': - df = pd.read_csv(file_path) - elif file_ext in ['.xlsx', '.xls']: - df = pd.read_excel(file_path) - elif file_ext == '.parquet': +def read_dataframe(path: str | Path, *, nrows: Optional[int] = None) -> pd.DataFrame: + file_path = Path(path) + file_ext = file_path.suffix.lower() + if file_ext == ".csv": + return pd.read_csv(file_path, nrows=nrows) + if file_ext in {".xlsx", ".xls"}: + return pd.read_excel(file_path, nrows=nrows) + if file_ext == ".parquet": df = pd.read_parquet(file_path) - else: - raise ValueError(f"Unsupported file type: {file_ext}") - + return df.head(nrows) if nrows is not None else df + raise ValueError("Unsupported file type") + + +def extract_file_metadata(file_path: str) -> Dict[str, Any]: + df = read_dataframe(file_path) row_count, column_count = df.shape - + if column_count == 0: + raise ValueError("Dataset has no columns") + columns_info = [] - for col in df.columns: - non_null_count = int(df[col].count()) - null_count = int(df[col].isna().sum()) - dtype = str(df[col].dtype) - - columns_info.append({ - "name": str(col), - "dtype": dtype, - "non_null_count": non_null_count, - "null_count": null_count - }) - + for column in df.columns: + series = df[column] + columns_info.append( + { + "name": str(column), + "dtype": str(series.dtype), + "non_null_count": int(series.count()), + "null_count": int(series.isna().sum()), + } + ) + return { - "row_count": row_count, - "column_count": column_count, - "column_info": {"columns": columns_info} + "row_count": int(row_count), + "column_count": int(column_count), + "column_info": {"columns": columns_info}, } -def delete_file(storage_path: str) -> bool: - """Delete a file from the filesystem.""" - path = Path(storage_path) - if path.exists(): - path.unlink() - return True - return False +def read_dataset_preview(uri: str, total_rows: int, rows: int) -> Dict[str, Any]: + with artifact_store.materialize(uri) as local_path: + df = read_dataframe(local_path, nrows=rows) + df_filled = df.astype(object).where(pd.notna(df), None) + return { + "columns": [str(column) for column in df.columns], + "data": df_filled.values.tolist(), + "row_count": total_rows, + "preview_rows": len(df), + } + + +async def _delete_artifact_quietly(uri: str) -> None: + try: + await asyncio.to_thread(artifact_store.delete, uri) + except Exception as exc: + # An orphaned private artifact is preferable to rolling back an already + # committed database delete. Operators can clean these from provider logs. + print(f"[NoCodeML] Artifact cleanup warning: {type(exc).__name__}") diff --git a/Backend/app/services/download_naming.py b/Backend/app/services/download_naming.py new file mode 100644 index 0000000..b67666a --- /dev/null +++ b/Backend/app/services/download_naming.py @@ -0,0 +1,29 @@ +"""Safe, descriptive filenames for downloadable NoCodeML session artifacts.""" +from __future__ import annotations + +import re +from datetime import datetime, timezone +from pathlib import Path + + +def slugify(value: str, *, fallback: str = "dataset", max_length: int = 64) -> str: + text = Path(value or "").stem.lower().strip() + text = re.sub(r"[^a-z0-9]+", "-", text).strip("-") + return (text[:max_length].rstrip("-") or fallback) + + +def artifact_filename( + dataset_name: str, + artifact: str, + extension: str, + *, + timestamp: datetime | None = None, +) -> str: + moment = timestamp or datetime.now(timezone.utc) + stamp = moment.strftime("%Y%m%d-%H%M%S") + dataset_slug = slugify(dataset_name) + artifact_slug = slugify(artifact, fallback="export") + clean_extension = extension.lower().lstrip(".") + if not re.fullmatch(r"[a-z0-9]{1,12}", clean_extension): + raise ValueError("Invalid export file extension") + return f"nocodeml_{dataset_slug}_{artifact_slug}_{stamp}.{clean_extension}" diff --git a/Backend/app/services/eda_service.py b/Backend/app/services/eda_service.py index 789e36b..d74a422 100644 --- a/Backend/app/services/eda_service.py +++ b/Backend/app/services/eda_service.py @@ -1,545 +1,346 @@ -"""EDA service for exploratory data analysis operations.""" -import pandas as pd -import numpy as np +"""Exploratory data analysis for NoCodeML datasets.""" +from __future__ import annotations + +import asyncio from pathlib import Path -from typing import Dict, Any, List, Optional, Tuple +from typing import Any, Dict, List, Optional, Tuple from uuid import UUID -from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy import select + +import numpy as np +import pandas as pd from fastapi import HTTPException, status +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession from app.models.dataset import Dataset +from app.services.artifact_store import artifact_store -# Constants -SAMPLE_THRESHOLD = 10000 +SAMPLE_THRESHOLD = 10_000 RANDOM_SEED = 42 MAX_SAMPLE_VALUES = 5 +MAX_CATEGORY_TRACES = 20 + + +def _read_dataframe(path: Path) -> pd.DataFrame: + extension = path.suffix.lower() + if extension == ".csv": + return pd.read_csv(path) + if extension in {".xlsx", ".xls"}: + return pd.read_excel(path) + if extension == ".parquet": + return pd.read_parquet(path) + raise ValueError(f"Unsupported dataset format: {extension}") + +def _read_artifact_dataframe(uri: str) -> pd.DataFrame: + with artifact_store.materialize(uri) as local_path: + return _read_dataframe(local_path) -async def load_dataset(dataset_id: UUID, user_id: int, db: AsyncSession) -> Tuple[pd.DataFrame, Dataset]: - """Load dataset from storage and verify ownership.""" - query = select(Dataset).where( - Dataset.id == dataset_id, - Dataset.user_id == user_id + +async def load_dataset( + dataset_id: UUID, + user_id: int, + db: AsyncSession, +) -> Tuple[pd.DataFrame, Dataset]: + """Load a user-owned dataset from local or private object storage.""" + result = await db.execute( + select(Dataset).where(Dataset.id == dataset_id, Dataset.user_id == user_id) ) - result = await db.execute(query) dataset = result.scalar_one_or_none() - if not dataset: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Dataset not found or access denied" - ) - - storage_path = Path(dataset.storage_path) - if not storage_path.exists(): + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Dataset not found or access denied") + + try: + df = await asyncio.to_thread(_read_artifact_dataframe, dataset.storage_path) + except FileNotFoundError as exc: raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="Dataset file not found on storage" - ) - - try: - # Read file based on extension - file_ext = storage_path.suffix.lower() - if file_ext == '.csv': - df = pd.read_csv(storage_path) - elif file_ext in ['.xlsx', '.xls']: - df = pd.read_excel(storage_path) - elif file_ext == '.parquet': - df = pd.read_parquet(storage_path) - else: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Unsupported file format: {file_ext}" - ) - - return df, dataset - except Exception as e: - if isinstance(e, HTTPException): - raise e + detail="Dataset artifact is missing from NoCodeML storage.", + ) from exc + except ValueError as exc: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc + except Exception as exc: raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Failed to read dataset: {str(e)}" - ) + detail="Dataset could not be read from NoCodeML storage.", + ) from exc + + return df, dataset + + +def _is_row_sequence(series: pd.Series) -> bool: + """Detect a simple 0..N-1 or 1..N row-number column without flagging arbitrary unique numerics.""" + if len(series) < 2 or series.isna().any() or not pd.api.types.is_integer_dtype(series): + return False + values = series.to_numpy(dtype=np.int64, copy=True) + if len(np.unique(values)) != len(values): + return False + sorted_values = np.sort(values) + start = int(sorted_values[0]) + if start not in {0, 1}: + return False + expected = np.arange(start, start + len(sorted_values), dtype=np.int64) + return bool(np.array_equal(sorted_values, expected)) def detect_id_columns(df: pd.DataFrame) -> List[str]: - """Auto-detect ID columns based on name patterns and uniqueness.""" - id_columns = [] - - for col in df.columns: - col_lower = col.lower() - - # Name-based detection - if any(pattern in col_lower for pattern in ['id', '_id', 'id_', 'index', 'key']): - id_columns.append(col) - continue - - # Uniqueness-based detection (100% unique and numeric/string type) - if df[col].nunique() == len(df): - if df[col].dtype in ['int64', 'int32', 'object', 'string']: - id_columns.append(col) - - return id_columns + """Conservatively identify identifier columns. + + V2 treated every fully-unique numeric/string column as an ID. That can discard + valid continuous features and targets. V3 only trusts explicit identifier-style + names or true row-number sequences. + """ + detected: List[str] = [] + explicit_names = {"id", "index", "key", "uuid", "guid", "rowid", "row_id"} + + for column in df.columns: + name = str(column) + lowered = name.strip().lower() + name_match = ( + lowered in explicit_names + or lowered.endswith("_id") + or lowered.startswith("id_") + or lowered.endswith("_key") + ) + sequence_match = _is_row_sequence(df[column]) + if name_match or sequence_match: + detected.append(name) + return detected + + +def _native_sample_values(series: pd.Series) -> List[Any]: + values: List[Any] = [] + for value in series.dropna().head(MAX_SAMPLE_VALUES).tolist(): + if isinstance(value, (np.integer,)): + values.append(int(value)) + elif isinstance(value, (np.floating,)): + values.append(float(value)) + elif isinstance(value, (np.bool_,)): + values.append(bool(value)) + else: + values.append(str(value)) + return values def get_column_info(df: pd.DataFrame, id_columns: List[str]) -> List[Dict[str, Any]]: - """Extract detailed information for each column.""" - columns_info = [] - - for col in df.columns: - missing_count = int(df[col].isna().sum()) - missing_percent = (missing_count / len(df)) * 100 - unique_count = int(df[col].nunique()) - - # Get sample values (non-null) - sample_values = df[col].dropna().head(MAX_SAMPLE_VALUES).tolist() - - # Convert numpy types to native Python types - sample_values = [ - int(x) if isinstance(x, (np.integer, np.int64)) else - float(x) if isinstance(x, (np.floating, np.float64)) else - str(x) - for x in sample_values - ] - - columns_info.append({ - 'name': col, - 'dtype': str(df[col].dtype), - 'missing_count': missing_count, - 'missing_percent': round(missing_percent, 2), - 'unique_count': unique_count, - 'is_id_column': col in id_columns, - 'sample_values': sample_values - }) - - return columns_info - - -def categorize_columns(df: pd.DataFrame, id_columns: List[str]) -> Tuple[List[str], List[str]]: - """Categorize columns into numeric and categorical (keeping ID columns for now).""" - numeric_cols = [] - categorical_cols = [] - - for col in df.columns: - # Keep ID columns in the analysis - user requested this - if pd.api.types.is_numeric_dtype(df[col]): - numeric_cols.append(col) - else: - categorical_cols.append(col) - - return numeric_cols, categorical_cols + row_count = len(df) + info: List[Dict[str, Any]] = [] + for column in df.columns: + missing = int(df[column].isna().sum()) + info.append( + { + "name": str(column), + "dtype": str(df[column].dtype), + "missing_count": missing, + "missing_percent": round((missing / row_count) * 100, 2) if row_count else 0.0, + "unique_count": int(df[column].nunique(dropna=True)), + "is_id_column": str(column) in id_columns, + "sample_values": _native_sample_values(df[column]), + } + ) + return info + + +def categorize_columns(df: pd.DataFrame) -> Tuple[List[str], List[str]]: + numeric = [str(column) for column in df.columns if pd.api.types.is_numeric_dtype(df[column])] + categorical = [str(column) for column in df.columns if str(column) not in numeric] + return numeric, categorical def compute_statistics(df: pd.DataFrame, numeric_columns: List[str]) -> Dict[str, Any]: - """Compute statistical summary for numeric columns.""" if not numeric_columns: return {} - - stats_df = df[numeric_columns].describe() - - # Convert to dictionary with proper type conversion - statistics = {} - for col in numeric_columns: - col_stats = {} - for stat_name in stats_df.index: - value = stats_df.loc[stat_name, col] - if pd.notna(value): - col_stats[stat_name] = float(value) - else: - col_stats[stat_name] = None - statistics[col] = col_stats - + described = df[numeric_columns].describe(percentiles=[0.25, 0.5, 0.75]) + statistics: Dict[str, Any] = {} + for column in numeric_columns: + statistics[column] = { + key: (float(value) if pd.notna(value) else None) + for key, value in described[column].items() + } return statistics def compute_correlations(df: pd.DataFrame, numeric_columns: List[str]) -> Optional[Dict[str, Any]]: - """Compute correlation matrix for numeric columns.""" if len(numeric_columns) < 2: return None - - try: - corr_matrix = df[numeric_columns].corr() - - # Convert to dictionary format - correlations = { - 'columns': numeric_columns, - 'matrix': corr_matrix.values.tolist(), - 'pairs': [] - } - - # Extract strong correlations (|r| > 0.7, excluding diagonal) - for i, col1 in enumerate(numeric_columns): - for j, col2 in enumerate(numeric_columns): - if i < j: # Avoid duplicates - corr_value = corr_matrix.iloc[i, j] - if abs(corr_value) > 0.7: - correlations['pairs'].append({ - 'col1': col1, - 'col2': col2, - 'correlation': round(float(corr_value), 3) - }) - - return correlations - except Exception: - return None + matrix = df[numeric_columns].corr() + matrix_values = [ + [float(value) if pd.notna(value) else None for value in row] + for row in matrix.to_numpy() + ] + pairs: List[Dict[str, Any]] = [] + for index, first in enumerate(numeric_columns): + for second_index in range(index + 1, len(numeric_columns)): + value = matrix.iloc[index, second_index] + if pd.notna(value) and abs(float(value)) >= 0.7: + pairs.append( + { + "col1": first, + "col2": numeric_columns[second_index], + "correlation": round(float(value), 3), + } + ) + return {"columns": numeric_columns, "matrix": matrix_values, "pairs": pairs} def compute_missing_data_summary(df: pd.DataFrame) -> Dict[str, Any]: - """Compute summary of missing data.""" - total_cells = df.shape[0] * df.shape[1] + total_cells = int(df.shape[0] * df.shape[1]) total_missing = int(df.isna().sum().sum()) - missing_percent = (total_missing / total_cells) * 100 if total_cells > 0 else 0 - - columns_with_missing = [] - for col in df.columns: - missing_count = int(df[col].isna().sum()) - if missing_count > 0: - columns_with_missing.append({ - 'column': col, - 'missing_count': missing_count, - 'missing_percent': round((missing_count / len(df)) * 100, 2) - }) - - # Sort by missing count descending - columns_with_missing.sort(key=lambda x: x['missing_count'], reverse=True) - + columns = [] + for column in df.columns: + missing = int(df[column].isna().sum()) + if missing: + columns.append( + { + "column": str(column), + "missing_count": missing, + "missing_percent": round((missing / len(df)) * 100, 2) if len(df) else 0.0, + } + ) + columns.sort(key=lambda item: item["missing_count"], reverse=True) return { - 'total_missing': total_missing, - 'total_cells': total_cells, - 'missing_percent': round(missing_percent, 2), - 'columns_with_missing': columns_with_missing + "total_missing": total_missing, + "total_cells": total_cells, + "missing_percent": round((total_missing / total_cells) * 100, 2) if total_cells else 0.0, + "columns_with_missing": columns, } def get_preview_data(df: pd.DataFrame, max_rows: int = 100) -> Dict[str, Any]: - """ - Get preview data for the dataset with pagination metadata. - - Args: - df: DataFrame to preview - max_rows: Maximum number of rows to include (default 100) - - Returns: - Dictionary with columns, rows, total_rows, and page_size - """ - preview_df = df.head(max_rows) - - # Convert DataFrame to list of dictionaries - # Handle NaN/None values and convert numpy types - rows = [] - for _, row in preview_df.iterrows(): - row_dict = {} - for col in df.columns: - value = row[col] - - # Handle missing values - if pd.isna(value): - row_dict[col] = None - # Convert numpy types to Python native types - elif isinstance(value, (np.integer, np.int64)): - row_dict[col] = int(value) - elif isinstance(value, (np.floating, np.float64)): - # Round to 2 decimal places for display - row_dict[col] = round(float(value), 2) + preview = df.head(max_rows).astype(object).where(pd.notna(df.head(max_rows)), None) + rows: List[Dict[str, Any]] = [] + for record in preview.to_dict(orient="records"): + converted = {} + for key, value in record.items(): + if isinstance(value, np.integer): + converted[str(key)] = int(value) + elif isinstance(value, np.floating): + converted[str(key)] = float(value) + elif isinstance(value, np.bool_): + converted[str(key)] = bool(value) else: - row_dict[col] = str(value) - - rows.append(row_dict) - + converted[str(key)] = value + rows.append(converted) return { - 'columns': df.columns.tolist(), - 'rows': rows, - 'total_rows': len(df), - 'page_size': max_rows + "columns": [str(column) for column in df.columns], + "rows": rows, + "total_rows": len(df), + "page_size": len(rows), } async def get_eda_summary(dataset_id: UUID, user_id: int, db: AsyncSession) -> Dict[str, Any]: - """ - Generate comprehensive EDA summary for a dataset. - - Returns: - Dictionary containing dataset info, column info, statistics, correlations, - missing data summary, and preview data. - """ - # Load dataset df, dataset = await load_dataset(dataset_id, user_id, db) - - # Detect ID columns id_columns = detect_id_columns(df) - - # Categorize columns - numeric_columns, categorical_columns = categorize_columns(df, id_columns) - - # Get column information - columns_info = get_column_info(df, id_columns) - - # Compute statistics - statistics = compute_statistics(df, numeric_columns) - - # Compute correlations - correlations = compute_correlations(df, numeric_columns) - - # Missing data summary - missing_data_summary = compute_missing_data_summary(df) - - # Get preview data (first 100 rows for pagination) - preview_data = get_preview_data(df, max_rows=100) - - # Dataset info - dataset_info = { - 'id': str(dataset.id), - 'name': dataset.name, - 'row_count': len(df), - 'column_count': len(df.columns), - 'file_size_bytes': dataset.file_size_bytes, - 'file_name': dataset.file_name, - 'memory_usage_bytes': int(df.memory_usage(deep=True).sum()) - } - + numeric_columns, categorical_columns = categorize_columns(df) + return { - 'dataset_info': dataset_info, - 'columns': columns_info, - 'numeric_columns': numeric_columns, - 'categorical_columns': categorical_columns, - 'id_columns': id_columns, - 'statistics': statistics, - 'correlations': correlations, - 'missing_data_summary': missing_data_summary, - 'preview_data': preview_data + "dataset_info": { + "id": str(dataset.id), + "name": dataset.name, + "row_count": len(df), + "column_count": len(df.columns), + "file_size_bytes": dataset.file_size_bytes, + "file_name": dataset.file_name, + "memory_usage_bytes": int(df.memory_usage(deep=True).sum()), + }, + "columns": get_column_info(df, id_columns), + "numeric_columns": numeric_columns, + "categorical_columns": categorical_columns, + "id_columns": id_columns, + "statistics": compute_statistics(df, numeric_columns), + "correlations": compute_correlations(df, numeric_columns), + "missing_data_summary": compute_missing_data_summary(df), + "preview_data": get_preview_data(df, 100), } def sample_dataframe(df: pd.DataFrame) -> Tuple[pd.DataFrame, bool, int, int]: - """Sample dataframe if it exceeds threshold.""" - total_rows = len(df) - - if total_rows > SAMPLE_THRESHOLD: - df_sampled = df.sample(n=SAMPLE_THRESHOLD, random_state=RANDOM_SEED) - return df_sampled, True, total_rows, SAMPLE_THRESHOLD - - return df, False, total_rows, total_rows - - -def create_histogram(df: pd.DataFrame, column: str) -> Dict[str, Any]: - """Generate histogram plot data.""" - data = df[column].dropna() - - trace = { - 'x': data.tolist(), - 'type': 'histogram', - 'name': column, - 'marker': { - 'color': 'rgba(99, 102, 241, 0.7)', - 'line': {'color': 'rgba(99, 102, 241, 1)', 'width': 1} - }, - 'autobinx': True - } - - # Calculate mean and median for reference lines - mean_val = float(data.mean()) - median_val = float(data.median()) - - layout = { - 'title': f'Distribution of {column}', - 'xaxis': {'title': column}, - 'yaxis': {'title': 'Frequency'}, - 'shapes': [ - { - 'type': 'line', - 'x0': mean_val, - 'x1': mean_val, - 'y0': 0, - 'y1': 1, - 'yref': 'paper', - 'line': {'color': 'red', 'width': 2, 'dash': 'dash'} - }, - { - 'type': 'line', - 'x0': median_val, - 'x1': median_val, - 'y0': 0, - 'y1': 1, - 'yref': 'paper', - 'line': {'color': 'green', 'width': 2, 'dash': 'dash'} - } - ], - 'annotations': [ - { - 'x': mean_val, - 'y': 0.95, - 'yref': 'paper', - 'text': f'Mean: {mean_val:.2f}', - 'showarrow': False, - 'font': {'color': 'red'} - }, - { - 'x': median_val, - 'y': 0.85, - 'yref': 'paper', - 'text': f'Median: {median_val:.2f}', - 'showarrow': False, - 'font': {'color': 'green'} - } - ] - } - - return {'data': [trace], 'layout': layout} + total = len(df) + if total > SAMPLE_THRESHOLD: + sampled = df.sample(n=SAMPLE_THRESHOLD, random_state=RANDOM_SEED) + return sampled, True, total, len(sampled) + return df, False, total, total + + +def _require_column(df: pd.DataFrame, column: Optional[str], *, numeric: bool = False) -> str: + if not column or column not in df.columns: + raise HTTPException(status_code=400, detail=f"Column '{column}' was not found in the dataset") + if numeric and not pd.api.types.is_numeric_dtype(df[column]): + raise HTTPException(status_code=400, detail=f"Column '{column}' must be numeric for this plot") + return column + + +def _histogram(df: pd.DataFrame, column: str) -> tuple[List[Dict[str, Any]], Dict[str, Any]]: + values = df[column].dropna().astype(float).tolist() + return ( + [{"x": values, "type": "histogram", "name": column, "opacity": 0.85}], + {"title": f"Distribution of {column}", "xaxis": {"title": column}, "yaxis": {"title": "Count"}}, + ) -def create_scatter(df: pd.DataFrame, x_column: str, y_column: str, group_by: Optional[str] = None) -> Dict[str, Any]: - """Generate scatter plot data.""" - # Remove rows with missing values in relevant columns - cols_to_check = [x_column, y_column] +def _scatter(df: pd.DataFrame, x_column: str, y_column: str, group_by: Optional[str]) -> tuple[List[Dict[str, Any]], Dict[str, Any]]: + columns = [x_column, y_column] + ([group_by] if group_by else []) + clean = df[columns].dropna() + traces: List[Dict[str, Any]] = [] if group_by: - cols_to_check.append(group_by) - - df_clean = df[cols_to_check].dropna() - - if group_by and group_by in df_clean.columns: - # Group by category - traces = [] - for category in df_clean[group_by].unique(): - mask = df_clean[group_by] == category - traces.append({ - 'x': df_clean.loc[mask, x_column].tolist(), - 'y': df_clean.loc[mask, y_column].tolist(), - 'type': 'scatter', - 'mode': 'markers', - 'name': str(category), - 'marker': {'size': 8} - }) - data = traces + categories = clean[group_by].astype(str).value_counts().head(MAX_CATEGORY_TRACES).index + for category in categories: + mask = clean[group_by].astype(str) == category + traces.append( + { + "x": clean.loc[mask, x_column].astype(float).tolist(), + "y": clean.loc[mask, y_column].astype(float).tolist(), + "type": "scatter", + "mode": "markers", + "name": str(category), + } + ) else: - # Single scatter plot - data = [{ - 'x': df_clean[x_column].tolist(), - 'y': df_clean[y_column].tolist(), - 'type': 'scatter', - 'mode': 'markers', - 'marker': { - 'size': 8, - 'color': 'rgba(99, 102, 241, 0.7)', - 'line': {'color': 'rgba(99, 102, 241, 1)', 'width': 1} - } - }] - - # Calculate correlation - try: - correlation = df_clean[[x_column, y_column]].corr().iloc[0, 1] - corr_text = f'Correlation: {correlation:.3f}' - except Exception: - corr_text = '' - - layout = { - 'title': f'{y_column} vs {x_column}', - 'xaxis': {'title': x_column}, - 'yaxis': {'title': y_column}, - 'annotations': [ + traces.append( { - 'x': 0.05, - 'y': 0.95, - 'xref': 'paper', - 'yref': 'paper', - 'text': corr_text, - 'showarrow': False, - 'font': {'size': 12} + "x": clean[x_column].astype(float).tolist(), + "y": clean[y_column].astype(float).tolist(), + "type": "scatter", + "mode": "markers", + "name": f"{x_column} vs {y_column}", } - ] if corr_text else [] - } - - return {'data': data, 'layout': layout} - - -def create_box_plot(df: pd.DataFrame, column: str, group_by: Optional[str] = None) -> Dict[str, Any]: - """Generate box plot data.""" - if group_by and group_by in df.columns: - # Group by category - traces = [] - for category in df[group_by].dropna().unique(): - mask = df[group_by] == category - traces.append({ - 'y': df.loc[mask, column].dropna().tolist(), - 'type': 'box', - 'name': str(category), - 'boxmean': 'sd' - }) - data = traces - title = f'{column} by {group_by}' + ) + return traces, {"title": f"{x_column} vs {y_column}", "xaxis": {"title": x_column}, "yaxis": {"title": y_column}} + + +def _box(df: pd.DataFrame, column: str, group_by: Optional[str]) -> tuple[List[Dict[str, Any]], Dict[str, Any]]: + traces: List[Dict[str, Any]] = [] + if group_by: + clean = df[[column, group_by]].dropna() + categories = clean[group_by].astype(str).value_counts().head(MAX_CATEGORY_TRACES).index + for category in categories: + values = clean.loc[clean[group_by].astype(str) == category, column].astype(float).tolist() + traces.append({"y": values, "type": "box", "name": str(category), "boxpoints": "outliers"}) else: - # Single box plot - data = [{ - 'y': df[column].dropna().tolist(), - 'type': 'box', - 'name': column, - 'marker': {'color': 'rgba(99, 102, 241, 0.7)'}, - 'boxmean': 'sd' - }] - title = f'Box Plot of {column}' - - layout = { - 'title': title, - 'yaxis': {'title': column} - } - - return {'data': data, 'layout': layout} + traces.append({"y": df[column].dropna().astype(float).tolist(), "type": "box", "name": column, "boxpoints": "outliers"}) + return traces, {"title": f"Box plot of {column}", "yaxis": {"title": column}} + + +def _correlation(df: pd.DataFrame) -> tuple[List[Dict[str, Any]], Dict[str, Any]]: + numeric = [str(column) for column in df.columns if pd.api.types.is_numeric_dtype(df[column])] + if len(numeric) < 2: + raise HTTPException(status_code=400, detail="At least two numeric columns are required for correlation") + matrix = df[numeric].corr() + z = [[float(value) if pd.notna(value) else None for value in row] for row in matrix.to_numpy()] + return ( + [{"z": z, "x": numeric, "y": numeric, "type": "heatmap", "zmin": -1, "zmax": 1}], + {"title": "Correlation matrix"}, + ) -def create_correlation_heatmap(df: pd.DataFrame, numeric_columns: List[str]) -> Dict[str, Any]: - """Generate correlation heatmap data.""" - if len(numeric_columns) < 2: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Need at least 2 numeric columns for correlation heatmap" - ) - - corr_matrix = df[numeric_columns].corr() - - data = [{ - 'z': corr_matrix.values.tolist(), - 'x': numeric_columns, - 'y': numeric_columns, - 'type': 'heatmap', - 'colorscale': 'RdBu', - 'zmid': 0, - 'zmin': -1, - 'zmax': 1, - 'colorbar': {'title': 'Correlation'} - }] - - layout = { - 'title': 'Correlation Heatmap', - 'xaxis': {'title': '', 'tickangle': -45}, - 'yaxis': {'title': ''}, - 'height': 500 + len(numeric_columns) * 20 - } - - return {'data': data, 'layout': layout} - - -def create_bar_chart(df: pd.DataFrame, column: str) -> Dict[str, Any]: - """Generate bar chart for categorical column.""" - value_counts = df[column].value_counts().head(20) # Top 20 categories - - data = [{ - 'x': value_counts.index.tolist(), - 'y': value_counts.values.tolist(), - 'type': 'bar', - 'marker': { - 'color': 'rgba(99, 102, 241, 0.7)', - 'line': {'color': 'rgba(99, 102, 241, 1)', 'width': 1} - } - }] - - layout = { - 'title': f'Distribution of {column}', - 'xaxis': {'title': column, 'tickangle': -45}, - 'yaxis': {'title': 'Count'} - } - - return {'data': data, 'layout': layout} +def _bar(df: pd.DataFrame, column: str) -> tuple[List[Dict[str, Any]], Dict[str, Any]]: + counts = df[column].fillna("(missing)").astype(str).value_counts().head(50) + return ( + [{"x": counts.index.tolist(), "y": counts.astype(int).tolist(), "type": "bar", "name": column}], + {"title": f"Top values in {column}", "xaxis": {"title": column}, "yaxis": {"title": "Count"}}, + ) async def generate_plot_data( @@ -549,80 +350,37 @@ async def generate_plot_data( y_column: Optional[str], group_by: Optional[str], user_id: int, - db: AsyncSession + db: AsyncSession, ) -> Dict[str, Any]: - """ - Generate plot data based on plot type and column selections. - - Args: - dataset_id: UUID of the dataset - plot_type: Type of plot (histogram, scatter, box, correlation, bar) - x_column: Column for x-axis - y_column: Column for y-axis (optional) - group_by: Column to group by (optional) - user_id: User ID for authorization - db: Database session - - Returns: - Dictionary with plot data, layout, and sampling information - """ - # Load dataset - df, dataset = await load_dataset(dataset_id, user_id, db) - - # Sample if necessary - df_plot, is_sampled, total_rows, displayed_rows = sample_dataframe(df) - - # Validate columns - if x_column not in df.columns: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Column '{x_column}' not found in dataset" - ) - - if y_column and y_column not in df.columns: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Column '{y_column}' not found in dataset" - ) - - # Generate plot based on type - try: - if plot_type == 'histogram': - plot_result = create_histogram(df_plot, x_column) - elif plot_type == 'scatter': - if not y_column: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Scatter plot requires both x_column and y_column" - ) - plot_result = create_scatter(df_plot, x_column, y_column, group_by) - elif plot_type == 'box': - plot_result = create_box_plot(df_plot, x_column, group_by) - elif plot_type == 'correlation': - # For correlation, use all numeric columns (excluding IDs) - id_columns = detect_id_columns(df) - numeric_columns, _ = categorize_columns(df, id_columns) - plot_result = create_correlation_heatmap(df_plot, numeric_columns) - elif plot_type == 'bar': - plot_result = create_bar_chart(df_plot, x_column) - else: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Unsupported plot type: {plot_type}" - ) - - return { - 'data': plot_result['data'], - 'layout': plot_result['layout'], - 'is_sampled': is_sampled, - 'total_rows': total_rows, - 'displayed_rows': displayed_rows, - 'plot_type': plot_type - } - except Exception as e: - if isinstance(e, HTTPException): - raise e - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Failed to generate plot: {str(e)}" - ) + df, _dataset = await load_dataset(dataset_id, user_id, db) + sampled, is_sampled, total_rows, displayed_rows = sample_dataframe(df) + kind = plot_type.strip().lower() + + if kind == "histogram": + x = _require_column(sampled, x_column, numeric=True) + data, layout = _histogram(sampled, x) + elif kind == "scatter": + x = _require_column(sampled, x_column, numeric=True) + y = _require_column(sampled, y_column, numeric=True) + group = _require_column(sampled, group_by) if group_by else None + data, layout = _scatter(sampled, x, y, group) + elif kind == "box": + x = _require_column(sampled, x_column, numeric=True) + group = _require_column(sampled, group_by) if group_by else None + data, layout = _box(sampled, x, group) + elif kind == "correlation": + data, layout = _correlation(sampled) + elif kind == "bar": + x = _require_column(sampled, x_column) + data, layout = _bar(sampled, x) + else: + raise HTTPException(status_code=400, detail="Plot type must be histogram, scatter, box, correlation, or bar") + + return { + "data": data, + "layout": layout, + "is_sampled": is_sampled, + "total_rows": total_rows, + "displayed_rows": displayed_rows, + "plot_type": kind, + } diff --git a/Backend/app/services/model_trainer.py b/Backend/app/services/model_trainer.py index 429c6d7..7244181 100644 --- a/Backend/app/services/model_trainer.py +++ b/Backend/app/services/model_trainer.py @@ -1,464 +1,339 @@ -"""Model trainer for ML model training with AutoClean preprocessing.""" -import os -import uuid +"""Deterministic NoCodeML training pipeline. + +V3 persists preprocessing and the estimator together as one sklearn Pipeline. That +ensures predictions reuse the exact imputers, categorical encoders and scaling +fitted during training instead of reconstructing them with new category mappings. +""" +from __future__ import annotations + import time -import json -import pickle -import joblib +import uuid +from collections import Counter from pathlib import Path -from typing import Dict, Any, Tuple, Optional, List -import numpy as np +from typing import Any, Dict, List, Optional, Tuple -# Core ML libraries (always available) -from sklearn.model_selection import train_test_split, cross_val_score -from sklearn.preprocessing import StandardScaler, LabelEncoder +import joblib +import numpy as np +import pandas as pd +from sklearn.compose import ColumnTransformer +from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor +from sklearn.impute import SimpleImputer +from sklearn.linear_model import LinearRegression, LogisticRegression from sklearn.metrics import ( - accuracy_score, precision_score, recall_score, f1_score, - mean_squared_error, mean_absolute_error, r2_score, - classification_report, confusion_matrix, roc_auc_score + accuracy_score, + confusion_matrix, + f1_score, + mean_absolute_error, + mean_squared_error, + precision_score, + r2_score, + recall_score, + roc_auc_score, ) -from sklearn.linear_model import LogisticRegression, LinearRegression -from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor +from sklearn.model_selection import cross_val_score, train_test_split +from sklearn.pipeline import Pipeline +from sklearn.preprocessing import LabelEncoder, OneHotEncoder, StandardScaler -# Optional libraries with fallbacks -try: - import pandas as pd - PANDAS_AVAILABLE = True -except ImportError: - PANDAS_AVAILABLE = False - -try: - from autoclean import AutoClean - AUTOCLEAN_AVAILABLE = True -except ImportError: - AUTOCLEAN_AVAILABLE = False +from app.core.config import settings +from app.core.model_defaults import get_preprocessing_config, get_training_config +from app.services.artifact_store import artifact_store +from app.services.hyperparameter_optimizer import hyperparameter_optimizer try: from xgboost import XGBClassifier, XGBRegressor - XGBOOST_AVAILABLE = True -except ImportError: - XGBOOST_AVAILABLE = False +except ImportError: # pragma: no cover - dependency is installed in production + XGBClassifier = XGBRegressor = None try: from lightgbm import LGBMClassifier, LGBMRegressor - LIGHTGBM_AVAILABLE = True -except ImportError: - LIGHTGBM_AVAILABLE = False - -from app.core.model_defaults import get_preprocessing_config, get_training_config -from app.core.model_cache import get_model_info -from app.services.hyperparameter_optimizer import hyperparameter_optimizer +except ImportError: # pragma: no cover - dependency is installed in production + LGBMClassifier = LGBMRegressor = None class ModelTrainer: - """Handles ML model training with preprocessing and evaluation.""" - - def __init__(self, models_dir: str = "./models"): - """ - Initialize model trainer. - - Args: - models_dir: Directory to save trained models - """ - self.models_dir = Path(models_dir) + """Train and persist classification/regression pipelines.""" + + def __init__(self, models_dir: Optional[str] = None): + self.models_dir = Path(models_dir or settings.MODELS_DIR).expanduser() self.models_dir.mkdir(parents=True, exist_ok=True) - - # Model registry + self.model_registry = { - 'classification': { - 'LogisticRegression': LogisticRegression, - 'RandomForestClassifier': RandomForestClassifier, + "classification": { + "LogisticRegression": LogisticRegression, + "RandomForestClassifier": RandomForestClassifier, + }, + "regression": { + "LinearRegression": LinearRegression, + "RandomForestRegressor": RandomForestRegressor, }, - 'regression': { - 'LinearRegression': LinearRegression, - 'RandomForestRegressor': RandomForestRegressor, - } } - - # Add optional models if available - if XGBOOST_AVAILABLE: - self.model_registry['classification']['XGBClassifier'] = XGBClassifier - self.model_registry['regression']['XGBRegressor'] = XGBRegressor - - if LIGHTGBM_AVAILABLE: - self.model_registry['classification']['LGBMClassifier'] = LGBMClassifier - self.model_registry['regression']['LGBMRegressor'] = LGBMRegressor - - def load_dataset(self, dataset_path: str) -> Optional[Any]: - """ - Load dataset from file. - - Args: - dataset_path: Path to dataset file - - Returns: - Loaded dataset or None if pandas not available - """ - if not PANDAS_AVAILABLE: - raise ImportError("Pandas is required for dataset loading") - - if not os.path.exists(dataset_path): - raise FileNotFoundError(f"Dataset not found: {dataset_path}") - - file_ext = Path(dataset_path).suffix.lower() - - if file_ext == '.csv': - return pd.read_csv(dataset_path) - elif file_ext in ['.xlsx', '.xls']: - return pd.read_excel(dataset_path) - elif file_ext == '.parquet': - return pd.read_parquet(dataset_path) - else: - raise ValueError(f"Unsupported file format: {file_ext}") - - def preprocess_data(self, df: Any, target_column: str, config: Dict[str, Any], task_type: str = 'classification') -> Tuple[Any, Any, Optional[Any]]: - """ - Preprocess data using AutoClean or basic preprocessing. - - Args: - df: Input dataframe - target_column: Name of target column - config: Preprocessing configuration - task_type: 'classification' or 'regression' - - Returns: - Tuple of (features, target, label_encoder) - label_encoder is None for regression - """ - if not PANDAS_AVAILABLE: - raise ImportError("Pandas is required for data preprocessing") - - # Separate features and target + if XGBClassifier is not None: + self.model_registry["classification"]["XGBClassifier"] = XGBClassifier + self.model_registry["regression"]["XGBRegressor"] = XGBRegressor + if LGBMClassifier is not None: + self.model_registry["classification"]["LGBMClassifier"] = LGBMClassifier + self.model_registry["regression"]["LGBMRegressor"] = LGBMRegressor + + @staticmethod + def _read_dataframe(path: Path) -> pd.DataFrame: + extension = path.suffix.lower() + if extension == ".csv": + return pd.read_csv(path) + if extension in {".xlsx", ".xls"}: + return pd.read_excel(path) + if extension == ".parquet": + return pd.read_parquet(path) + raise ValueError(f"Unsupported file format: {extension}") + + def load_dataset(self, dataset_uri: str) -> pd.DataFrame: + with artifact_store.materialize(dataset_uri) as local_path: + return self._read_dataframe(local_path) + + @staticmethod + def _select_features( + df: pd.DataFrame, + target_column: str, + selected_features: Optional[List[str]], + ) -> Tuple[pd.DataFrame, pd.Series]: if target_column not in df.columns: raise ValueError(f"Target column '{target_column}' not found in dataset") - - X = df.drop(columns=[target_column]) - y = df[target_column] - - if AUTOCLEAN_AVAILABLE and config.get('use_autoclean', True): - # Use AutoClean for preprocessing - try: - # Create a temporary dataframe with target for AutoClean - temp_df = df.copy() - - pipeline = AutoClean( - temp_df, - target=target_column, - duplicates=config.get('duplicates', True), - missing_num=config.get('missing_num', 'auto'), - missing_categ=config.get('missing_categ', 'auto'), - encode_categ=config.get('encode_categ', ['onehot']), - outliers=config.get('outliers', 'auto'), - extract_datetime=config.get('extract_datetime', False) - ) - - cleaned_df = pipeline.output - X_clean = cleaned_df.drop(columns=[target_column]) - y_clean = cleaned_df[target_column] - - # Encode target variable for classification if it's categorical - label_encoder = None - if task_type == 'classification' and y_clean.dtype == 'object': - label_encoder = LabelEncoder() - y_clean = pd.Series(label_encoder.fit_transform(y_clean), index=y_clean.index) - - return X_clean, y_clean, label_encoder - - except Exception as e: - print(f"AutoClean failed, falling back to basic preprocessing: {e}") - - # Basic preprocessing fallback - # Handle missing values - X_numeric = X.select_dtypes(include=[np.number]) - X_categorical = X.select_dtypes(exclude=[np.number]) - - # Fill numeric missing values with median - if not X_numeric.empty: - X_numeric = X_numeric.fillna(X_numeric.median()) - - # Fill categorical missing values with mode - if not X_categorical.empty: - for col in X_categorical.columns: - X_categorical[col] = X_categorical[col].fillna(X_categorical[col].mode()[0] if len(X_categorical[col].mode()) > 0 else 'unknown') - - # Simple label encoding for categorical variables - le = LabelEncoder() - for col in X_categorical.columns: - X_categorical[col] = le.fit_transform(X_categorical[col].astype(str)) - - # Combine back - if not X_numeric.empty and not X_categorical.empty: - X_processed = pd.concat([X_numeric, X_categorical], axis=1) - elif not X_numeric.empty: - X_processed = X_numeric + + if selected_features: + features = [feature for feature in selected_features if feature != target_column] + missing = [feature for feature in features if feature not in df.columns] + if missing: + raise ValueError(f"Selected features not found: {', '.join(missing[:10])}") else: - X_processed = X_categorical - - # Handle target missing values - y_processed = y.dropna() - X_processed = X_processed.loc[y_processed.index] - - # Encode target variable for classification if it's categorical (string/object type) - label_encoder = None - if task_type == 'classification' and y_processed.dtype == 'object': - label_encoder = LabelEncoder() - y_processed = pd.Series(label_encoder.fit_transform(y_processed), index=y_processed.index) - - return X_processed, y_processed, label_encoder - - def split_and_scale_data( - self, - X: Any, - y: Any, - config: Dict[str, Any], - task_type: str - ) -> Tuple[Any, Any, Any, Any, Optional[Any]]: - """ - Split data into train/test and apply scaling if needed. - - Args: - X: Features - y: Target - config: Training configuration - task_type: 'classification' or 'regression' - - Returns: - Tuple of (X_train, X_test, y_train, y_test, scaler) - """ - test_size = config.get('test_size', 0.2) - random_state = config.get('random_state', 42) - stratify = y if (task_type == 'classification' and config.get('stratify', True)) else None - - # Split data - X_train, X_test, y_train, y_test = train_test_split( - X, y, - test_size=test_size, - random_state=random_state, - stratify=stratify - ) - - # Apply scaling if needed - scaler = None - if config.get('scaling', True): - scaler = StandardScaler() - # Fit scaler on training data only (no data leakage) - X_train = scaler.fit_transform(X_train) - X_test = scaler.transform(X_test) - - return X_train, X_test, y_train, y_test, scaler - - def train_model( - self, - model_type: str, + features = [str(column) for column in df.columns if str(column) != target_column] + + if not features: + raise ValueError("At least one feature is required for training") + + X = df[features].copy() + y = df[target_column].copy() + + valid_target = y.notna() + X = X.loc[valid_target].reset_index(drop=True) + y = y.loc[valid_target].reset_index(drop=True) + if len(X) < 4: + raise ValueError("Not enough rows with a target value to train a model") + return X, y + + @staticmethod + def _prepare_target(y: pd.Series, task_type: str) -> Tuple[pd.Series, Optional[LabelEncoder]]: + if task_type == "classification": + encoder = LabelEncoder() + encoded = encoder.fit_transform(y.astype(str)) + if len(encoder.classes_) < 2: + raise ValueError("Classification requires at least two target classes") + return pd.Series(encoded, index=y.index), encoder + + if task_type == "regression": + numeric = pd.to_numeric(y, errors="coerce") + if numeric.isna().any(): + raise ValueError("Regression target must contain numeric values") + return numeric.astype(float), None + + raise ValueError("Task type must be classification or regression") + + @staticmethod + def _build_preprocessor(X: pd.DataFrame, scaling: bool = True) -> ColumnTransformer: + numeric_columns = [str(column) for column in X.columns if pd.api.types.is_numeric_dtype(X[column])] + categorical_columns = [str(column) for column in X.columns if str(column) not in numeric_columns] + + transformers = [] + if numeric_columns: + numeric_steps: List[Tuple[str, Any]] = [("imputer", SimpleImputer(strategy="median"))] + if scaling: + numeric_steps.append(("scaler", StandardScaler())) + transformers.append(("numeric", Pipeline(numeric_steps), numeric_columns)) + + if categorical_columns: + categorical_pipeline = Pipeline( + [ + ("imputer", SimpleImputer(strategy="most_frequent")), + ("encoder", OneHotEncoder(handle_unknown="ignore", sparse_output=True)), + ] + ) + transformers.append(("categorical", categorical_pipeline, categorical_columns)) + + if not transformers: + raise ValueError("No usable feature columns were found") + + return ColumnTransformer(transformers=transformers, remainder="drop", sparse_threshold=0.3) + + def _build_estimator(self, model_type: str, task_type: str, hyperparameters: Dict[str, Any]): + if task_type not in self.model_registry or model_type not in self.model_registry[task_type]: + raise ValueError(f"Model '{model_type}' is not available for {task_type}") + return self.model_registry[task_type][model_type](**hyperparameters) + + @staticmethod + def _safe_split( + X: pd.DataFrame, + y: pd.Series, task_type: str, - X_train: Any, - y_train: Any, - hyperparameters: Dict[str, Any] - ) -> Any: - """ - Train a model with given hyperparameters. - - Args: - model_type: Type of model to train - task_type: 'classification' or 'regression' - X_train: Training features - y_train: Training target - hyperparameters: Model hyperparameters - - Returns: - Trained model - """ - if task_type not in self.model_registry: - raise ValueError(f"Unsupported task type: {task_type}") - - if model_type not in self.model_registry[task_type]: - raise ValueError(f"Model '{model_type}' not available for {task_type}") - - model_class = self.model_registry[task_type][model_type] - model = model_class(**hyperparameters) - - # Train the model - start_time = time.time() - model.fit(X_train, y_train) - training_time = time.time() - start_time - - return model, training_time - + test_size: float, + random_state: int, + ): + test_size = min(0.4, max(0.1, float(test_size))) + stratify = None + if task_type == "classification": + counts = Counter(y.tolist()) + if counts and min(counts.values()) >= 2: + stratify = y + try: + return train_test_split( + X, + y, + test_size=test_size, + random_state=random_state, + stratify=stratify, + ) + except ValueError: + return train_test_split( + X, + y, + test_size=test_size, + random_state=random_state, + stratify=None, + ) + + @staticmethod + def _cv_folds(y_train: pd.Series, task_type: str, requested: int) -> int: + requested = max(2, min(int(requested), 5)) + if task_type == "classification": + counts = Counter(y_train.tolist()) + return max(0, min(requested, min(counts.values()) if counts else 0)) + return min(requested, len(y_train)) if len(y_train) >= 2 else 0 + + @staticmethod def evaluate_model( - self, - model: Any, - X_train: Any, - X_test: Any, - y_train: Any, - y_test: Any, + pipeline: Pipeline, + X_train: pd.DataFrame, + X_test: pd.DataFrame, + y_train: pd.Series, + y_test: pd.Series, task_type: str, - cv_folds: int = 3, - label_encoder: Optional[Any] = None + cv_folds: int, + label_encoder: Optional[LabelEncoder], ) -> Dict[str, Any]: - """ - Evaluate trained model and calculate metrics. - - Args: - model: Trained model - X_train: Training features - X_test: Test features - y_train: Training target - y_test: Test target - task_type: 'classification' or 'regression' - cv_folds: Number of cross-validation folds - - Returns: - Dictionary of evaluation metrics with separate train/test scores - """ - # Make predictions on both train and test sets - y_train_pred = model.predict(X_train) - y_test_pred = model.predict(X_test) - - metrics = { - 'train': {}, - 'test': {}, - 'confusion_matrix': None, - 'class_labels': None - } - - if task_type == 'classification': - # Training set metrics - metrics['train']['accuracy'] = float(accuracy_score(y_train, y_train_pred)) - metrics['train']['precision'] = float(precision_score(y_train, y_train_pred, average='weighted', zero_division=0)) - metrics['train']['recall'] = float(recall_score(y_train, y_train_pred, average='weighted', zero_division=0)) - metrics['train']['f1_score'] = float(f1_score(y_train, y_train_pred, average='weighted', zero_division=0)) - - # Test set metrics - metrics['test']['accuracy'] = float(accuracy_score(y_test, y_test_pred)) - metrics['test']['precision'] = float(precision_score(y_test, y_test_pred, average='weighted', zero_division=0)) - metrics['test']['recall'] = float(recall_score(y_test, y_test_pred, average='weighted', zero_division=0)) - metrics['test']['f1_score'] = float(f1_score(y_test, y_test_pred, average='weighted', zero_division=0)) - - # ROC AUC for binary classification - try: - if len(np.unique(y_test)) == 2: - y_pred_proba = model.predict_proba(X_test)[:, 1] - metrics['test']['roc_auc'] = float(roc_auc_score(y_test, y_pred_proba)) - except: - pass # Skip if not supported - - # Confusion matrix with labels - cm = confusion_matrix(y_test, y_test_pred) - unique_labels = sorted(np.unique(y_test).tolist()) - - # If we have a label encoder, use the original string labels + train_prediction = pipeline.predict(X_train) + test_prediction = pipeline.predict(X_test) + metrics: Dict[str, Any] = {"train": {}, "test": {}} + + if task_type == "classification": + for name, truth, prediction in ( + ("train", y_train, train_prediction), + ("test", y_test, test_prediction), + ): + metrics[name] = { + "accuracy": float(accuracy_score(truth, prediction)), + "precision": float(precision_score(truth, prediction, average="weighted", zero_division=0)), + "recall": float(recall_score(truth, prediction, average="weighted", zero_division=0)), + "f1_score": float(f1_score(truth, prediction, average="weighted", zero_division=0)), + } + + estimator = pipeline.named_steps["model"] + if hasattr(estimator, "predict_proba") and len(np.unique(y_test)) == 2: + try: + probabilities = pipeline.predict_proba(X_test)[:, 1] + metrics["test"]["roc_auc"] = float(roc_auc_score(y_test, probabilities)) + except Exception: + pass + + labels = sorted(np.unique(np.concatenate([np.asarray(y_test), np.asarray(test_prediction)])).tolist()) if label_encoder is not None: - original_labels = label_encoder.inverse_transform(unique_labels) - label_strings = [str(label) for label in original_labels] + display_labels = [str(value) for value in label_encoder.inverse_transform(np.asarray(labels, dtype=int))] else: - label_strings = [str(label) for label in unique_labels] - - metrics['confusion_matrix'] = { - 'matrix': cm.tolist(), - 'labels': label_strings + display_labels = [str(value) for value in labels] + metrics["confusion_matrix"] = { + "matrix": confusion_matrix(y_test, test_prediction, labels=labels).tolist(), + "labels": display_labels, } - - else: # regression - # Training set metrics - metrics['train']['mse'] = float(mean_squared_error(y_train, y_train_pred)) - metrics['train']['rmse'] = float(np.sqrt(metrics['train']['mse'])) - metrics['train']['mae'] = float(mean_absolute_error(y_train, y_train_pred)) - metrics['train']['r2_score'] = float(r2_score(y_train, y_train_pred)) - - # Test set metrics - metrics['test']['mse'] = float(mean_squared_error(y_test, y_test_pred)) - metrics['test']['rmse'] = float(np.sqrt(metrics['test']['mse'])) - metrics['test']['mae'] = float(mean_absolute_error(y_test, y_test_pred)) - metrics['test']['r2_score'] = float(r2_score(y_test, y_test_pred)) - - # Cross-validation scores - try: - cv_scores = cross_val_score(model, X_train, y_train, cv=cv_folds, n_jobs=-1) - metrics['cv_scores'] = cv_scores.tolist() - metrics['mean_cv_score'] = float(cv_scores.mean()) - metrics['std_cv_score'] = float(cv_scores.std()) - except Exception as e: - print(f"Cross-validation failed: {e}") - + else: + for name, truth, prediction in ( + ("train", y_train, train_prediction), + ("test", y_test, test_prediction), + ): + mse = float(mean_squared_error(truth, prediction)) + metrics[name] = { + "mse": mse, + "rmse": float(np.sqrt(mse)), + "mae": float(mean_absolute_error(truth, prediction)), + "r2_score": float(r2_score(truth, prediction)), + } + + folds = ModelTrainer._cv_folds(y_train, task_type, cv_folds) + if folds >= 2: + try: + scores = cross_val_score(pipeline, X_train, y_train, cv=folds, n_jobs=1) + metrics["cv_scores"] = [float(value) for value in scores] + metrics["mean_cv_score"] = float(scores.mean()) + metrics["std_cv_score"] = float(scores.std()) + except Exception as exc: + metrics["cv_warning"] = f"Cross-validation unavailable: {type(exc).__name__}" + return metrics - - def get_feature_importance(self, model: Any, feature_names: List[str]) -> Optional[Dict[str, float]]: - """ - Extract feature importance from model if supported. - - Args: - model: Trained model - feature_names: List of feature names - - Returns: - Dictionary of feature importances or None - """ + + @staticmethod + def get_feature_importance(pipeline: Pipeline) -> Optional[Dict[str, List[Any]]]: try: - if hasattr(model, 'feature_importances_'): - importances = model.feature_importances_ - elif hasattr(model, 'coef_'): - importances = np.abs(model.coef_).flatten() + preprocessor: ColumnTransformer = pipeline.named_steps["preprocessor"] + estimator = pipeline.named_steps["model"] + feature_names = [ + str(name).replace("numeric__", "").replace("categorical__", "") + for name in preprocessor.get_feature_names_out() + ] + + if hasattr(estimator, "feature_importances_"): + values = np.asarray(estimator.feature_importances_, dtype=float) + elif hasattr(estimator, "coef_"): + coefficients = np.asarray(estimator.coef_, dtype=float) + values = np.abs(coefficients).mean(axis=0) if coefficients.ndim > 1 else np.abs(coefficients) else: return None - - # Create feature importance dictionary - feature_importance = { - name: float(importance) - for name, importance in zip(feature_names, importances) + + length = min(len(feature_names), len(values)) + pairs = sorted( + zip(feature_names[:length], values[:length]), + key=lambda item: float(item[1]), + reverse=True, + ) + return { + "features": [name for name, _value in pairs], + "importance": [float(value) for _name, value in pairs], } - - # Sort by importance - feature_importance = dict(sorted( - feature_importance.items(), - key=lambda x: x[1], - reverse=True - )) - - return feature_importance - - except Exception as e: - print(f"Failed to extract feature importance: {e}") + except Exception: return None - - def save_model(self, model: Any, model_id: str, scaler: Optional[Any] = None, - label_encoder: Optional[Any] = None) -> str: - """ - Save trained model, scaler, and label encoder to disk. - - Args: - model: Trained model - model_id: Unique identifier for the model - scaler: Fitted scaler (optional) - label_encoder: Fitted label encoder for classification (optional) - - Returns: - Path to saved model file - """ - model_path = self.models_dir / f"{model_id}.joblib" - - # Save model, scaler, and label_encoder together - model_data = { - 'model': model, - 'scaler': scaler, - 'label_encoder': label_encoder, - 'saved_at': time.time() + + def save_model( + self, + pipeline: Pipeline, + model_id: str, + label_encoder: Optional[LabelEncoder], + feature_columns: List[str], + ) -> str: + safe_id = "".join(character for character in model_id if character.isalnum() or character in {"-", "_"}) + safe_id = safe_id or str(uuid.uuid4()) + local_path = self.models_dir / f"{safe_id}.joblib" + artifact = { + "model": pipeline, + "label_encoder": label_encoder, + "feature_columns": feature_columns, + "saved_at": time.time(), + "artifact_version": 3, } - - joblib.dump(model_data, model_path) - return str(model_path) - - def load_model(self, model_path: str) -> Tuple[Any, Optional[Any]]: - """ - Load saved model and scaler. - - Args: - model_path: Path to saved model file - - Returns: - Tuple of (model, scaler) - """ - model_data = joblib.load(model_path) - return model_data['model'], model_data.get('scaler') - + joblib.dump(artifact, local_path) + + uri = artifact_store.put_file( + local_path, + f"models/{safe_id}.joblib", + "application/octet-stream", + ) + if artifact_store.is_remote: + local_path.unlink(missing_ok=True) + return uri + def train_complete_pipeline( self, dataset_path: str, @@ -471,157 +346,108 @@ def train_complete_pipeline( selected_features: Optional[List[str]] = None, feature_types: Optional[Dict[str, str]] = None, job_id: Optional[str] = None, - enable_optimization: bool = False + enable_optimization: bool = False, ) -> Dict[str, Any]: - """ - Complete training pipeline from data loading to model saving. - - Args: - dataset_path: Path to dataset file - target_column: Name of target column - model_type: Type of model to train - task_type: 'classification' or 'regression' - hyperparameters: Model hyperparameters - preprocessing_config: Preprocessing configuration - training_config: Training configuration - selected_features: List of feature column names to use (filters dataset) - feature_types: Dictionary mapping feature names to types ('categorical' or 'numerical') - job_id: Job ID for model saving - - Returns: - Dictionary with training results - """ - # Use default configs if not provided + """Run the complete, reusable training pipeline.""" + del feature_types # Dtypes are derived from the actual selected dataframe. preprocessing_config = preprocessing_config or get_preprocessing_config() training_config = training_config or get_training_config() - - start_time = time.time() - + started = time.time() + try: - # Load dataset df = self.load_dataset(dataset_path) - - # Filter to selected features if specified - if selected_features: - # Include target column + selected features - columns_to_keep = selected_features + [target_column] - # Only keep columns that exist in the dataframe - columns_to_keep = [col for col in columns_to_keep if col in df.columns] - df = df[columns_to_keep] - - # Preprocess data - X, y, label_encoder = self.preprocess_data(df, target_column, preprocessing_config, task_type) - - # Split and scale data - X_train, X_test, y_train, y_test, scaler = self.split_and_scale_data( - X, y, training_config, task_type + X, raw_target = self._select_features(df, target_column, selected_features) + y, label_encoder = self._prepare_target(raw_target, task_type) + + test_size = training_config.get("test_size", 0.2) + random_state = int(training_config.get("random_state", 42)) + X_train, X_test, y_train, y_test = self._safe_split( + X, y, task_type, test_size, random_state ) - - # Calculate class imbalance for classification (if optimization enabled) - class_imbalance_ratio = None - if task_type == 'classification' and enable_optimization: - try: - from collections import Counter - class_counts = Counter(y_train) - if len(class_counts) == 2: - majority = max(class_counts.values()) - minority = min(class_counts.values()) - class_imbalance_ratio = majority / minority if minority > 0 else None - except Exception: - pass - - # Get optimized hyperparameters if enabled + + final_hyperparameters = dict(hyperparameters or {}) tuning_metadata = None if enable_optimization: - print(f"๐Ÿ” Optimizing hyperparameters for {model_type}...") - optimization_result = hyperparameter_optimizer.optimize( + imbalance = None + if task_type == "classification": + counts = Counter(y_train.tolist()) + if len(counts) == 2 and min(counts.values()) > 0: + imbalance = max(counts.values()) / min(counts.values()) + + optimized = hyperparameter_optimizer.optimize( model_type=model_type, task_type=task_type, n_samples=len(X_train), - n_features=X_train.shape[1] if hasattr(X_train, 'shape') else len(X_train.columns), - class_imbalance_ratio=class_imbalance_ratio, - user_params=hyperparameters + n_features=X_train.shape[1], + class_imbalance_ratio=imbalance, + user_params=final_hyperparameters, ) - - final_hyperparameters = optimization_result['params'] - applied_rules = optimization_result.get('applied_rules', []) - tuning_metadata = optimization_result['metadata'] - - print(f"โœ… Optimization complete!") - print(f"๐Ÿ“Š Optimized parameters: {final_hyperparameters}") - print(f"๐Ÿ“‹ Rules applied: {len(applied_rules)}") - else: - final_hyperparameters = hyperparameters - - # Train model - model, training_time = self.train_model( - model_type, task_type, X_train, y_train, final_hyperparameters + final_hyperparameters = optimized["params"] + tuning_metadata = optimized.get("metadata") + + preprocessor = self._build_preprocessor( + X_train, + scaling=bool(training_config.get("scaling", True)), + ) + estimator = self._build_estimator(model_type, task_type, final_hyperparameters) + pipeline = Pipeline( + [ + ("preprocessor", preprocessor), + ("model", estimator), + ] ) - - # Evaluate model + + training_started = time.time() + pipeline.fit(X_train, y_train) + training_time = time.time() - training_started + metrics = self.evaluate_model( - model, X_train, X_test, y_train, y_test, task_type, - training_config.get('cv_folds', 3), - label_encoder=label_encoder + pipeline, + X_train, + X_test, + y_train, + y_test, + task_type, + int(training_config.get("cv_folds", 3)), + label_encoder, ) - - # Get feature importance - feature_names = X.columns.tolist() if hasattr(X, 'columns') else [f'feature_{i}' for i in range(X.shape[1])] - feature_importance_dict = self.get_feature_importance(model, feature_names) - - # Format feature importance for frontend visualization - feature_importance = None - if feature_importance_dict: - # Convert dict to sorted lists for easy plotting - sorted_features = sorted(feature_importance_dict.items(), key=lambda x: x[1], reverse=True) - feature_importance = { - 'features': [f[0] for f in sorted_features], - 'importance': [f[1] for f in sorted_features] - } - - # Finalize tuning metadata with actual results (if optimization was enabled) - if enable_optimization and tuning_metadata is None: - # Generate metadata with actual test score - test_score = metrics.get('test', {}).get('accuracy' if task_type == 'classification' else 'r2_score', 0.0) - tuning_metadata = hyperparameter_optimizer.finalize_metadata( - optimized_params=final_hyperparameters, - test_score=test_score, - n_samples=len(X_train), - n_features=X_train.shape[1] if hasattr(X_train, 'shape') else len(X_train.columns), - model_type=model_type, - applied_rules=applied_rules if 'applied_rules' in dir() else None - ) - - # Save model with scaler and label_encoder + feature_importance = self.get_feature_importance(pipeline) + model_id = job_id or str(uuid.uuid4()) - model_path = self.save_model(model, model_id, scaler, label_encoder) - - total_time = time.time() - start_time - + model_uri = self.save_model( + pipeline, + model_id, + label_encoder, + [str(column) for column in X.columns], + ) + return { - 'success': True, - 'model_path': model_path, - 'metrics': metrics, - 'feature_importance': feature_importance, - 'confusion_matrix': metrics.get('confusion_matrix'), # Already formatted in evaluate_model - 'training_time_seconds': training_time, - 'total_time_seconds': total_time, - 'dataset_info': { - 'total_samples': len(X), - 'train_samples': len(X_train), - 'test_samples': len(X_test), - 'n_features': X.shape[1], - 'feature_names': feature_names + "success": True, + "model_path": model_uri, + "metrics": metrics, + "feature_importance": feature_importance, + "confusion_matrix": metrics.get("confusion_matrix"), + "training_time_seconds": training_time, + "training_time": training_time, + "total_time_seconds": time.time() - started, + "dataset_info": { + "total_samples": len(X), + "train_samples": len(X_train), + "test_samples": len(X_test), + "n_features": X.shape[1], + "feature_names": [str(column) for column in X.columns], + }, + "preprocessing_config": { + **preprocessing_config, + "implementation": "fitted_sklearn_pipeline", }, - 'preprocessing_config': preprocessing_config, - 'training_config': training_config, - 'hyperparameters': final_hyperparameters, - 'hyperparameter_tuning': tuning_metadata + "training_config": training_config, + "hyperparameters": final_hyperparameters, + "hyperparameter_tuning": tuning_metadata, } - - except Exception as e: + except Exception as exc: return { - 'success': False, - 'error': str(e), - 'training_time_seconds': time.time() - start_time + "success": False, + "error": str(exc)[:500], + "training_time_seconds": time.time() - started, } diff --git a/Backend/app/services/prediction_service.py b/Backend/app/services/prediction_service.py index 0823efc..609ff83 100644 --- a/Backend/app/services/prediction_service.py +++ b/Backend/app/services/prediction_service.py @@ -1,325 +1,255 @@ -"""Prediction service for loading models and making predictions.""" +"""Prediction service using persisted V3 training pipelines.""" +from __future__ import annotations + +import asyncio +import io +import uuid +from pathlib import Path +from typing import Any, Dict, List, Optional + import joblib -import pandas as pd import numpy as np -from pathlib import Path -from typing import Dict, Any, List, Optional -from fastapi import HTTPException, UploadFile -import uuid -import io +import pandas as pd +from fastapi import HTTPException, UploadFile, status +from sqlalchemy import and_, desc, select +from app.core.config import settings from app.db.sync_session import SyncSessionLocal -from app.models.training import TrainingRun from app.models.prediction import PredictionBatch +from app.models.training import TrainingRun +from app.services.artifact_store import artifact_store + + +MAX_BATCH_FILE_SIZE = 100 * 1024 * 1024 class PredictionService: - """Service for making predictions with trained models.""" - def __init__(self): - self.models_dir = Path("./models") - self.predictions_dir = Path("./predictions") + self.predictions_dir = Path(settings.PREDICTIONS_DIR).expanduser() self.predictions_dir.mkdir(parents=True, exist_ok=True) - - def _preprocess_features(self, df: pd.DataFrame) -> pd.DataFrame: - """ - Apply the same preprocessing that was done during training. - This includes encoding categorical variables with label encoding. - - Args: - df: DataFrame with raw feature values - - Returns: - DataFrame with preprocessed (encoded) features - """ - from sklearn.preprocessing import LabelEncoder - - # Separate numeric and categorical - X_numeric = df.select_dtypes(include=[np.number]) - X_categorical = df.select_dtypes(exclude=[np.number]) - - # Fill numeric missing values with median - if not X_numeric.empty: - X_numeric = X_numeric.fillna(X_numeric.median()) - - # Fill categorical missing values with mode and encode - if not X_categorical.empty: - for col in X_categorical.columns: - # Fill missing values - mode_val = X_categorical[col].mode() - fill_val = mode_val[0] if len(mode_val) > 0 else 'unknown' - X_categorical[col] = X_categorical[col].fillna(fill_val) - - # Label encode each categorical column - le = LabelEncoder() - X_categorical[col] = le.fit_transform(X_categorical[col].astype(str)) - - # Combine back in original column order - if not X_numeric.empty and not X_categorical.empty: - # Preserve original column order - result = pd.DataFrame(index=df.index) - for col in df.columns: - if col in X_numeric.columns: - result[col] = X_numeric[col] - elif col in X_categorical.columns: - result[col] = X_categorical[col] - return result - elif not X_numeric.empty: - return X_numeric - else: - return X_categorical - + + @staticmethod + def _required_features(model_data: Dict[str, Any], training_config: Dict[str, Any]) -> List[str]: + features = model_data.get("feature_columns") or training_config.get("selectedFeatures") or [] + return [str(feature) for feature in features] + + @staticmethod + def _decode_predictions(predictions: Any, label_encoder: Optional[Any]) -> np.ndarray: + values = np.asarray(predictions) + if label_encoder is None: + return values + return np.asarray(label_encoder.inverse_transform(values.astype(int))) + + @staticmethod + def _probability_labels(model: Any, label_encoder: Optional[Any]) -> List[str]: + estimator = model.named_steps.get("model") if hasattr(model, "named_steps") else model + classes = np.asarray(getattr(estimator, "classes_", [])) + if classes.size == 0: + return [] + if label_encoder is not None: + try: + return [str(value) for value in label_encoder.inverse_transform(classes.astype(int))] + except Exception: + pass + return [str(value) for value in classes] + async def predict_single(self, experiment_id: uuid.UUID, features: Dict[str, Any]) -> Dict[str, Any]: - """ - Make a single prediction. - - Args: - experiment_id: Experiment ID - features: Dictionary of feature values - - Returns: - Prediction result with confidence scores - """ - # Load best model from experiment - model_data = self._load_best_model(experiment_id) - model = model_data['model'] - scaler = model_data.get('scaler') - label_encoder = model_data.get('label_encoder') - training_config = model_data.get('config', {}) - - # Get expected feature columns - if scaler and hasattr(scaler, 'feature_names_in_'): - feature_columns = list(scaler.feature_names_in_) - elif 'selectedFeatures' in training_config: - feature_columns = training_config['selectedFeatures'] - else: - # Use all provided features + model_data = await asyncio.to_thread(self._load_best_model, experiment_id) + model = model_data["model"] + label_encoder = model_data.get("label_encoder") + training_config = model_data.get("config", {}) + feature_columns = self._required_features(model_data, training_config) + + if not feature_columns: feature_columns = list(features.keys()) - - # Prepare input data with correct feature order + missing = [column for column in feature_columns if column not in features] + if missing: + raise HTTPException(status_code=400, detail=f"Missing required features: {missing}") + + frame = pd.DataFrame([{column: features[column] for column in feature_columns}]) try: - feature_values = [features[col] for col in feature_columns] - df = pd.DataFrame([feature_values], columns=feature_columns) - except KeyError as e: - missing_cols = [col for col in feature_columns if col not in features] + raw_prediction = model.predict(frame) + except Exception as exc: raise HTTPException( - status_code=400, - detail=f"Missing required features: {missing_cols}" - ) - - # Apply preprocessing (categorical encoding, missing value handling) - df_preprocessed = self._preprocess_features(df) - - # Apply scaling if exists - if scaler: - # Use .values to avoid feature name warnings - scaled_values = scaler.transform(df_preprocessed.values) - else: - scaled_values = df_preprocessed.values - - # Make prediction (using numpy array to match training) - prediction = model.predict(scaled_values)[0] - - # Get prediction probabilities if available + status_code=status.HTTP_400_BAD_REQUEST, + detail="The supplied feature values are incompatible with the trained model.", + ) from exc + + decoded = self._decode_predictions(raw_prediction, label_encoder) probabilities = None confidence = None - if hasattr(model, 'predict_proba'): - proba = model.predict_proba(scaled_values)[0] - probabilities = { - str(i): float(p) for i, p in enumerate(proba) - } - confidence = float(max(proba)) - - # Decode prediction if label encoder exists - if label_encoder: - prediction = label_encoder.inverse_transform([prediction])[0] - + if hasattr(model, "predict_proba"): + try: + values = np.asarray(model.predict_proba(frame)[0], dtype=float) + labels = self._probability_labels(model, label_encoder) + probabilities = { + (labels[index] if index < len(labels) else str(index)): float(probability) + for index, probability in enumerate(values) + } + confidence = float(values.max()) if values.size else None + except Exception: + probabilities = None + confidence = None + + prediction = decoded[0] + if isinstance(prediction, np.generic): + prediction = prediction.item() return { - 'prediction': str(prediction), - 'probabilities': probabilities, - 'confidence': confidence + "prediction": str(prediction), + "probabilities": probabilities, + "confidence": confidence, } - - async def predict_batch(self, experiment_id: uuid.UUID, file: UploadFile, user_id: int) -> Dict[str, Any]: - """ - Make batch predictions from CSV file and track ownership. - - Args: - experiment_id: Experiment ID - file: CSV file with features - user_id: User ID (integer) for ownership tracking - - Returns: - Dictionary with download URL for predictions CSV - """ - # Load best model and config - model_data = self._load_best_model(experiment_id) - model = model_data['model'] - scaler = model_data.get('scaler') - label_encoder = model_data.get('label_encoder') - training_config = model_data.get('config', {}) - - # Read uploaded CSV - content = await file.read() - df_original = pd.read_csv(io.BytesIO(content)) - - # Get feature columns from training config or scaler - if scaler and hasattr(scaler, 'feature_names_in_'): - feature_columns = list(scaler.feature_names_in_) - elif 'selectedFeatures' in training_config: - feature_columns = training_config['selectedFeatures'] - else: - # Fallback: use all columns except common non-feature columns - exclude_cols = ['customerID', 'id', 'ID', training_config.get('targetColumn', 'target')] - feature_columns = [col for col in df_original.columns if col not in exclude_cols] - - # Extract only the feature columns needed for prediction + + async def predict_batch( + self, + experiment_id: uuid.UUID, + file: UploadFile, + user_id: int, + ) -> Dict[str, Any]: try: - df_features = df_original[feature_columns].copy() - except KeyError as e: - missing_cols = [col for col in feature_columns if col not in df_original.columns] + content = await file.read(MAX_BATCH_FILE_SIZE + 1) + finally: + await file.close() + + if len(content) > MAX_BATCH_FILE_SIZE: raise HTTPException( - status_code=400, - detail=f"Missing required feature columns: {missing_cols}" + status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, + detail="Batch prediction CSV exceeds the 100 MB limit.", ) - - # Apply preprocessing (categorical encoding, missing value handling) - df_preprocessed = self._preprocess_features(df_features) - - # Apply scaling if exists - if scaler: - # Use .values to avoid feature name warnings (model was trained without feature names) - scaled_values = scaler.transform(df_preprocessed.values) - df_scaled = scaled_values - else: - df_scaled = df_preprocessed.values - - # Make predictions (using numpy arrays to match training) - predictions = model.predict(df_scaled) - - # Get probabilities if available - if hasattr(model, 'predict_proba'): - probabilities = model.predict_proba(df_scaled) - df_original['confidence'] = probabilities.max(axis=1) - - # Decode predictions if label encoder exists - if label_encoder: - predictions = label_encoder.inverse_transform(predictions) - - df_original['prediction'] = predictions - - # Save predictions to file (with all original columns + prediction + confidence) - prediction_id = str(uuid.uuid4()) - output_path = self.predictions_dir / f"{prediction_id}.csv" - df_original.to_csv(output_path, index=False) - - # Save to database for ownership tracking - db = SyncSessionLocal() + try: - prediction_batch = PredictionBatch( - id=uuid.UUID(prediction_id), - user_id=user_id, - experiment_id=experiment_id, - file_path=str(output_path), - total_predictions=len(df_original) + original = pd.read_csv(io.BytesIO(content)) + except Exception as exc: + raise HTTPException(status_code=400, detail="The batch prediction CSV could not be parsed.") from exc + if original.empty: + raise HTTPException(status_code=400, detail="The batch prediction CSV contains no rows.") + + model_data = await asyncio.to_thread(self._load_best_model, experiment_id) + model = model_data["model"] + label_encoder = model_data.get("label_encoder") + training_config = model_data.get("config", {}) + feature_columns = self._required_features(model_data, training_config) + if not feature_columns: + feature_columns = [str(column) for column in original.columns] + + missing = [column for column in feature_columns if column not in original.columns] + if missing: + raise HTTPException(status_code=400, detail=f"Missing required feature columns: {missing}") + + features = original[feature_columns].copy() + try: + raw_predictions = model.predict(features) + decoded = self._decode_predictions(raw_predictions, label_encoder) + except Exception as exc: + raise HTTPException( + status_code=400, + detail="The batch feature values are incompatible with the trained model.", + ) from exc + + result_frame = original.copy() + result_frame["prediction"] = decoded + if hasattr(model, "predict_proba"): + try: + result_frame["confidence"] = np.asarray(model.predict_proba(features), dtype=float).max(axis=1) + except Exception: + pass + + prediction_id = uuid.uuid4() + local_output = self.predictions_dir / f"{prediction_id}.csv" + result_frame.to_csv(local_output, index=False) + + artifact_uri: Optional[str] = None + try: + artifact_uri = await asyncio.to_thread( + artifact_store.put_file, + local_output, + f"predictions/{user_id}/{prediction_id}.csv", + "text/csv", ) - db.add(prediction_batch) - db.commit() - except Exception as e: - db.rollback() - # Log error but don't fail the prediction - print(f"Warning: Failed to save prediction batch to database: {e}") - finally: - db.close() - + if artifact_store.is_remote: + local_output.unlink(missing_ok=True) + + db = SyncSessionLocal() + try: + prediction_batch = PredictionBatch( + id=prediction_id, + user_id=user_id, + experiment_id=experiment_id, + file_path=artifact_uri, + total_predictions=len(result_frame), + ) + db.add(prediction_batch) + db.commit() + except Exception: + db.rollback() + raise + finally: + db.close() + except Exception as exc: + local_output.unlink(missing_ok=True) + if artifact_uri: + try: + await asyncio.to_thread(artifact_store.delete, artifact_uri) + except Exception: + pass + raise HTTPException( + status_code=500, + detail="Prediction results could not be persisted.", + ) from exc + return { - 'prediction_id': prediction_id, - 'total_predictions': len(df_original), - 'download_url': f'/api/v1/predictions/download/{prediction_id}' + "prediction_id": str(prediction_id), + "total_predictions": len(result_frame), + "download_url": f"/api/v1/predictions/download/{prediction_id}", } - + def _load_best_model(self, experiment_id: uuid.UUID) -> Dict[str, Any]: - """ - Load the best trained model for an experiment by querying the database. - - Args: - experiment_id: Experiment ID - - Returns: - Dictionary with model, scaler, and label_encoder - """ db = SyncSessionLocal() try: - # Query for completed training runs for this experiment - from sqlalchemy import select, and_, desc - - # Get the most recent completed training run query = ( select(TrainingRun) .filter( and_( TrainingRun.experiment_id == experiment_id, - TrainingRun.status == 'completed' + TrainingRun.status == "completed", ) ) .order_by(desc(TrainingRun.created_at)) ) - training_run = db.execute(query).scalars().first() - if not training_run: - raise HTTPException( - status_code=404, - detail="No trained models found for this experiment. Please train models first." - ) - - # Extract best model from run results - results = training_run.results - if not results or 'best_model' not in results: - raise HTTPException( - status_code=500, - detail="Training run completed but no best model found in results." - ) - - best_model_info = results['best_model'] - - # Find the full model data in the models array - models = results.get('models', []) - best_model_data = None - for model in models: - if model.get('model_type') == best_model_info.get('model_type'): - best_model_data = model - break - - if not best_model_data or 'model_path' not in best_model_data: - raise HTTPException( - status_code=500, - detail="Best model path not found in training results." - ) - - # Load model from file - model_path = Path(best_model_data['model_path']) - - if not model_path.exists(): - raise HTTPException( - status_code=404, - detail=f"Model file not found at: {model_path}. The model may have been deleted." - ) - - # Load model data (includes model, scaler, label_encoder) - model_data = joblib.load(model_path) - - # Include the training config in the returned data - model_data['config'] = training_run.config_snapshot - - return model_data - - except HTTPException: - raise - except Exception as e: - raise HTTPException( - status_code=500, - detail=f"Failed to load model: {str(e)}" + raise HTTPException(status_code=404, detail="No trained models found for this experiment.") + + results = training_run.results or {} + best_model_info = results.get("best_model") + if not best_model_info: + raise HTTPException(status_code=409, detail="The latest completed run has no successful best model.") + + best_model_data = next( + ( + model + for model in results.get("models", []) + if model.get("model_type") == best_model_info.get("model_type") and model.get("model_path") + ), + None, ) + if not best_model_data: + raise HTTPException(status_code=500, detail="Best-model artifact metadata is missing.") + + try: + with artifact_store.materialize(best_model_data["model_path"]) as local_path: + model_data = joblib.load(local_path) + except FileNotFoundError as exc: + raise HTTPException(status_code=404, detail="The trained model artifact is no longer available.") from exc + except HTTPException: + raise + except Exception as exc: + raise HTTPException(status_code=500, detail="The trained model artifact could not be loaded.") from exc + + if not isinstance(model_data, dict) or "model" not in model_data: + raise HTTPException(status_code=500, detail="The trained model artifact is invalid.") + + model_data["config"] = training_run.config_snapshot or {} + return model_data finally: db.close() diff --git a/Backend/app/services/session_manager.py b/Backend/app/services/session_manager.py new file mode 100644 index 0000000..6abcdeb --- /dev/null +++ b/Backend/app/services/session_manager.py @@ -0,0 +1,279 @@ +"""Temporary anonymous workspace management for NoCodeML guest sessions.""" +from __future__ import annotations + +import hashlib +import json +import secrets +import shutil +import threading +import time +from pathlib import Path +from typing import Any + +from app.core.config import settings + + +class SessionError(Exception): + """Base error for temporary session operations.""" + + +class InvalidSessionToken(SessionError): + pass + + +class SessionNotFound(SessionError): + pass + + +class SessionExpired(SessionError): + pass + + +class SessionManager: + """Owns isolated, short-lived filesystem workspaces for anonymous users. + + Raw session tokens never appear on disk. Each workspace directory uses a + SHA-256 digest of the token and contains only temporary NoCodeML artifacts. + """ + + WORKSPACE_DIRS = ("datasets", "analysis", "training", "models", "predictions", "exports") + META_FILE = ".session.json" + + def __init__( + self, + root: Path | None = None, + ttl_seconds: int | None = None, + close_grace_seconds: int | None = None, + ) -> None: + self.root = (root or settings.session_root).resolve() + self.ttl_seconds = ttl_seconds or settings.SESSION_TTL_MINUTES * 60 + self.close_grace_seconds = close_grace_seconds or settings.SESSION_CLOSE_GRACE_SECONDS + self._lock = threading.RLock() + + def ensure_root(self) -> Path: + self.root.mkdir(parents=True, exist_ok=True, mode=0o700) + return self.root + + @staticmethod + def _validate_token(token: str) -> str: + value = (token or "").strip() + if not 32 <= len(value) <= 128: + raise InvalidSessionToken("Invalid session token") + allowed = set("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_") + if any(char not in allowed for char in value): + raise InvalidSessionToken("Invalid session token") + return value + + @classmethod + def token_digest(cls, token: str) -> str: + value = cls._validate_token(token) + return hashlib.sha256(value.encode("utf-8")).hexdigest() + + def _workspace_for_token(self, token: str) -> Path: + digest = self.token_digest(token) + return self.root / digest + + def _metadata_path(self, workspace: Path) -> Path: + return workspace / self.META_FILE + + @staticmethod + def _now() -> int: + return int(time.time()) + + def _read_metadata(self, workspace: Path) -> dict[str, Any]: + metadata_path = self._metadata_path(workspace) + if not metadata_path.is_file(): + raise SessionNotFound("Session does not exist") + try: + payload = json.loads(metadata_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise SessionNotFound("Session metadata is unavailable") from exc + if not isinstance(payload, dict): + raise SessionNotFound("Session metadata is invalid") + payload.setdefault("active_jobs", 0) + return payload + + def _write_metadata(self, workspace: Path, metadata: dict[str, Any]) -> None: + metadata_path = self._metadata_path(workspace) + temp_path = workspace / f"{self.META_FILE}.tmp" + temp_path.write_text(json.dumps(metadata, separators=(",", ":")), encoding="utf-8") + temp_path.replace(metadata_path) + + def create(self) -> tuple[str, dict[str, Any]]: + self.ensure_root() + with self._lock: + while True: + token = secrets.token_urlsafe(32) + workspace = self._workspace_for_token(token) + if not workspace.exists(): + break + + workspace.mkdir(mode=0o700) + for name in self.WORKSPACE_DIRS: + (workspace / name).mkdir(mode=0o700) + + now = self._now() + metadata: dict[str, Any] = { + "created_at": now, + "last_seen": now, + "expires_at": now + self.ttl_seconds, + "delete_after": None, + "active_jobs": 0, + } + self._write_metadata(workspace, metadata) + return token, metadata.copy() + + def resolve(self, token: str, *, touch: bool = True) -> tuple[Path, dict[str, Any]]: + self.ensure_root() + workspace = self._workspace_for_token(token) + + with self._lock: + if not workspace.is_dir(): + raise SessionNotFound("Session does not exist") + + metadata = self._read_metadata(workspace) + now = self._now() + expires_at = int(metadata.get("expires_at") or 0) + delete_after = metadata.get("delete_after") + active_jobs = max(0, int(metadata.get("active_jobs") or 0)) + + if active_jobs <= 0: + if expires_at <= now: + self._delete_workspace(workspace) + raise SessionExpired("Session has expired") + if delete_after is not None and int(delete_after) <= now: + self._delete_workspace(workspace) + raise SessionExpired("Session has ended") + + if touch: + metadata["last_seen"] = now + metadata["expires_at"] = now + self.ttl_seconds + metadata["delete_after"] = None + self._write_metadata(workspace, metadata) + + return workspace, metadata.copy() + + def touch(self, token: str) -> dict[str, Any]: + _, metadata = self.resolve(token, touch=True) + return metadata + + def acquire_job(self, token: str) -> dict[str, Any]: + """Hold the workspace while a long-running ML job is active.""" + workspace, metadata = self.resolve(token, touch=True) + with self._lock: + metadata = self._read_metadata(workspace) + metadata["active_jobs"] = max(0, int(metadata.get("active_jobs") or 0)) + 1 + now = self._now() + metadata["last_seen"] = now + metadata["expires_at"] = now + self.ttl_seconds + self._write_metadata(workspace, metadata) + return metadata.copy() + + def release_job(self, token: str) -> None: + workspace = self._workspace_for_token(token) + with self._lock: + if not workspace.is_dir(): + return + try: + metadata = self._read_metadata(workspace) + except SessionNotFound: + return + metadata["active_jobs"] = max(0, int(metadata.get("active_jobs") or 0) - 1) + metadata["last_seen"] = self._now() + self._write_metadata(workspace, metadata) + + def reset_stale_job_leases(self) -> int: + """Clear job leases left by a previous API process. + + In-process training jobs cannot survive a backend restart, so any + persisted active_jobs count is necessarily stale on startup. Clearing + those counters lets normal close/TTL cleanup reclaim interrupted runs. + """ + self.ensure_root() + reset = 0 + with self._lock: + for workspace in list(self.root.iterdir()): + if not workspace.is_dir(): + continue + try: + metadata = self._read_metadata(workspace) + except SessionNotFound: + continue + if int(metadata.get("active_jobs") or 0) > 0: + metadata["active_jobs"] = 0 + self._write_metadata(workspace, metadata) + reset += 1 + return reset + + def mark_closing(self, token: str) -> None: + """Schedule deletion after a grace period. + + Browsers also fire unload/pagehide during refresh. A returning page can + therefore rescue the session simply by touching it before delete_after. + Active ML jobs hold a cleanup lease until they finish. + """ + workspace = self._workspace_for_token(token) + with self._lock: + if not workspace.is_dir(): + return + try: + metadata = self._read_metadata(workspace) + except SessionNotFound: + return + now = self._now() + metadata["last_seen"] = now + metadata["delete_after"] = now + self.close_grace_seconds + self._write_metadata(workspace, metadata) + + def delete(self, token: str) -> bool: + workspace = self._workspace_for_token(token) + with self._lock: + if not workspace.exists(): + return False + self._delete_workspace(workspace) + return True + + def _delete_workspace(self, workspace: Path) -> None: + resolved = workspace.resolve() + if resolved.parent != self.root: + raise SessionError("Refusing to delete a path outside the session root") + shutil.rmtree(resolved, ignore_errors=False) + + def safe_path(self, token: str, *parts: str, touch: bool = True) -> Path: + workspace, _ = self.resolve(token, touch=touch) + candidate = workspace.joinpath(*parts).resolve() + if candidate != workspace and workspace not in candidate.parents: + raise SessionError("Unsafe session artifact path") + return candidate + + def cleanup_expired(self) -> int: + """Delete expired, close-marked, or corrupt orphan workspaces.""" + self.ensure_root() + now = self._now() + removed = 0 + + with self._lock: + for workspace in list(self.root.iterdir()): + if not workspace.is_dir(): + continue + should_remove = False + try: + metadata = self._read_metadata(workspace) + active_jobs = max(0, int(metadata.get("active_jobs") or 0)) + expires_at = int(metadata.get("expires_at") or 0) + delete_after = metadata.get("delete_after") + should_remove = active_jobs <= 0 and ( + expires_at <= now + or (delete_after is not None and int(delete_after) <= now) + ) + except (SessionNotFound, TypeError, ValueError): + should_remove = True + + if should_remove: + self._delete_workspace(workspace) + removed += 1 + + return removed + + +session_manager = SessionManager() diff --git a/Backend/app/services/task_manager.py b/Backend/app/services/task_manager.py index a4b9796..b0a047b 100644 --- a/Backend/app/services/task_manager.py +++ b/Backend/app/services/task_manager.py @@ -1,326 +1,192 @@ -"""Task management service for handling Celery training tasks.""" -import uuid -from typing import Dict, Any, Optional, List +"""Celery task management for NoCodeML. + +V3 uses the run-based worker for the primary experiment flow. The legacy single-job +methods remain available for backward-compatible API endpoints. +""" +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional + from celery.result import AsyncResult -from sqlalchemy.orm import Session -from app.worker.celery_app import celery_app -from app.worker.tasks import train_model_task, cancel_training_task, test_task -# Avoid circular import by importing the task by name instead -# from app.worker.training_tasks import train_models -from app.models.training import TrainingJob, TrainingResult, TrainingLog, TrainingJobStatus -from app.models.experiment import Experiment, TrainingStatus from app.db.sync_session import get_sync_db -from app.core.model_cache import get_model_instance_config -from app.core.model_defaults import get_preprocessing_config, get_training_config +from app.models.training import TrainingJob, TrainingJobStatus, TrainingLog, TrainingResult +from app.worker.celery_app import celery_app +from app.worker.tasks import cancel_training_task, test_task class TaskManager: - """Manages Celery tasks for ML training.""" - + """Dispatch and inspect NoCodeML Celery tasks.""" + def __init__(self): - """Initialize task manager.""" self.celery_app = celery_app - + def start_training_task( self, job_id: str, experiment_id: str, dataset_id: str, model_type: str, - task_type: str + task_type: str, ) -> str: - """ - Start an asynchronous training task for a single model. - - Args: - job_id: Training job ID - experiment_id: Experiment ID - dataset_id: Dataset ID - model_type: Model type to train - task_type: 'classification' or 'regression' - - Returns: - Celery task ID - """ - # Start async task - train ONE model per task - # Use send_task to avoid circular import - print(f"[TaskManager] Dispatching task for job {job_id}, model: {model_type}") - task_result = self.celery_app.send_task( - 'app.worker.training_tasks.train_models', + """Dispatch one legacy job-based training task.""" + result = self.celery_app.send_task( + "app.worker.training_tasks.train_models", kwargs={ - 'job_id': job_id, - 'experiment_id': experiment_id, - 'dataset_id': dataset_id, - 'model_types': [model_type], # Single model as list - 'task_type': task_type - } + "job_id": job_id, + "experiment_id": experiment_id, + "dataset_id": dataset_id, + "model_types": [model_type], + "task_type": task_type, + }, ) - - print(f"[TaskManager] Task dispatched with ID: {task_result.id}") - return task_result.id - - def start_training_run_task( - self, - run_id: str, - experiment_id: str, - dataset_id: str - ) -> str: - """ - Start an asynchronous training run task (trains all models in config). - - Args: - run_id: Training run ID - experiment_id: Experiment ID - dataset_id: Dataset ID - - Returns: - Celery task ID - """ - print(f"[TaskManager] Dispatching training run task for run {run_id}") - task_result = self.celery_app.send_task( - 'app.worker.training_tasks.train_config_run', + return result.id + + def start_training_run_task(self, run_id: str, experiment_id: str, dataset_id: str) -> str: + """Dispatch the active V3 run worker.""" + result = self.celery_app.send_task( + "app.worker.run_tasks.train_config_run_v3", kwargs={ - 'run_id': run_id, - 'experiment_id': experiment_id, - 'dataset_id': dataset_id - } + "run_id": run_id, + "experiment_id": experiment_id, + "dataset_id": dataset_id, + }, ) - - print(f"[TaskManager] Training run task dispatched with ID: {task_result.id}") - return task_result.id - + return result.id + def get_task_status(self, task_id: str) -> Dict[str, Any]: - """ - Get status of a Celery task. - - Args: - task_id: Celery task ID - - Returns: - Task status information - """ try: result = AsyncResult(task_id, app=self.celery_app) - - status_info = { - 'task_id': task_id, - 'status': result.status, - 'ready': result.ready(), - 'successful': result.successful() if result.ready() else None, - 'failed': result.failed() if result.ready() else None, + payload: Dict[str, Any] = { + "task_id": task_id, + "status": result.status, + "ready": result.ready(), + "successful": result.successful() if result.ready() else None, + "failed": result.failed() if result.ready() else None, } - if result.ready(): - # Task is complete if result.successful(): - status_info['result'] = result.result + payload["result"] = result.result else: - status_info['error'] = str(result.result) if result.result else 'Unknown error' - status_info['traceback'] = getattr(result.result, 'traceback', None) + payload["error"] = str(result.result) if result.result else "Unknown error" + elif result.status == "PROGRESS": + payload["progress"] = result.info else: - # Task is still running, check for progress - if result.status == 'PROGRESS': - status_info['progress'] = result.info - else: - status_info['info'] = result.info - - return status_info - - except Exception as e: + payload["info"] = result.info + return payload + except Exception as exc: return { - 'task_id': task_id, - 'status': 'ERROR', - 'error': f'Failed to get task status: {str(e)}', - 'ready': False + "task_id": task_id, + "status": "ERROR", + "error": f"Failed to get task status: {exc}", + "ready": False, } - + def cancel_task(self, task_id: str) -> Dict[str, Any]: - """ - Cancel a running task. - - Args: - task_id: Celery task ID to cancel - - Returns: - Cancellation result - """ try: - # Revoke the task self.celery_app.control.revoke(task_id, terminate=True) - - # Also run the cancel task for cleanup - cancel_result = cancel_training_task.delay(task_id) - + cleanup = cancel_training_task.delay(task_id) return { - 'success': True, - 'message': f'Task {task_id} cancellation initiated', - 'task_id': task_id, - 'cancel_task_id': cancel_result.id + "success": True, + "message": f"Task {task_id} cancellation initiated", + "task_id": task_id, + "cancel_task_id": cleanup.id, } - - except Exception as e: - return { - 'success': False, - 'error': f'Failed to cancel task: {str(e)}', - 'task_id': task_id - } - + except Exception as exc: + return {"success": False, "error": f"Failed to cancel task: {exc}", "task_id": task_id} + def test_celery_connection(self) -> Dict[str, Any]: - """ - Test Celery connection with a simple task. - - Returns: - Test result - """ try: - # Send test task - task_result = test_task.delay("Celery connection test") - + result = test_task.delay("Celery connection test") return { - 'success': True, - 'message': 'Test task sent successfully', - 'task_id': task_result.id, - 'status': task_result.status + "success": True, + "message": "Test task sent successfully", + "task_id": result.id, + "status": result.status, } - - except Exception as e: - return { - 'success': False, - 'error': f'Celery connection test failed: {str(e)}' - } - + except Exception as exc: + return {"success": False, "error": f"Celery connection test failed: {exc}"} + def get_active_tasks(self) -> List[Dict[str, Any]]: - """ - Get list of active tasks. - - Returns: - List of active task information - """ try: - # Get active tasks from Celery - inspect = self.celery_app.control.inspect() - active_tasks = inspect.active() - - if not active_tasks: + active = self.celery_app.control.inspect().active() + if not active: return [] - - # Flatten task information - all_tasks = [] - for worker, tasks in active_tasks.items(): - for task in tasks: - all_tasks.append({ - 'worker': worker, - 'task_id': task['id'], - 'name': task['name'], - 'args': task.get('args', []), - 'kwargs': task.get('kwargs', {}), - 'time_start': task.get('time_start') - }) - - return all_tasks - - except Exception as e: - return [{ - 'error': f'Failed to get active tasks: {str(e)}' - }] + return [ + { + "worker": worker, + "task_id": task.get("id"), + "name": task.get("name"), + "args": task.get("args", []), + "kwargs": task.get("kwargs", {}), + "time_start": task.get("time_start"), + } + for worker, tasks in active.items() + for task in tasks + ] + except Exception as exc: + return [{"error": f"Failed to get active tasks: {exc}"}] def update_training_job_status_sync( job_id: str, status: TrainingJobStatus, celery_task_id: Optional[str] = None, - error_message: Optional[str] = None + error_message: Optional[str] = None, ) -> bool: - """ - Update training job status synchronously (for Celery tasks). - - Args: - job_id: Training job ID - status: New status - celery_task_id: Celery task ID - error_message: Error message if failed - - Returns: - True if updated successfully - """ + """Backward-compatible synchronous status helper for legacy workers.""" try: db_gen = get_sync_db() db = next(db_gen) - try: - # Find the job job = db.query(TrainingJob).filter(TrainingJob.id == job_id).first() if not job: - print(f"Training job {job_id} not found") return False - - # Update status job.status = status - if celery_task_id: job.celery_task_id = celery_task_id - if error_message: job.error_message = error_message - - # Update timestamps - from datetime import datetime if status == TrainingJobStatus.RUNNING: - job.started_at = datetime.utcnow() - elif status in [TrainingJobStatus.COMPLETED, TrainingJobStatus.FAILED, TrainingJobStatus.CANCELLED]: - job.completed_at = datetime.utcnow() - + job.started_at = datetime.now(timezone.utc) + elif status in { + TrainingJobStatus.COMPLETED, + TrainingJobStatus.FAILED, + TrainingJobStatus.CANCELLED, + }: + job.completed_at = datetime.now(timezone.utc) db.commit() return True - finally: db.close() - - except Exception as e: - print(f"Failed to update job status: {e}") + except Exception: return False -def save_training_result_sync( - job_id: str, - result_data: Dict[str, Any] -) -> bool: - """ - Save training results synchronously (for Celery tasks). - - Args: - job_id: Training job ID - result_data: Training result data - - Returns: - True if saved successfully - """ +def save_training_result_sync(job_id: str, result_data: Dict[str, Any]) -> bool: + """Backward-compatible result helper for legacy workers.""" try: db_gen = get_sync_db() db = next(db_gen) - try: - # Create training result - training_result = TrainingResult( - job_id=job_id, - model_path=result_data['model_path'], - metrics_json=result_data.get('metrics', {}), - feature_importance_json=result_data.get('feature_importance'), - confusion_matrix_json=result_data.get('metrics', {}).get('confusion_matrix'), - training_time_seconds=result_data.get('training_time_seconds', 0), - cross_val_scores=result_data.get('metrics', {}).get('cv_scores') + metrics = result_data.get("metrics", {}) + db.add( + TrainingResult( + job_id=job_id, + model_path=result_data["model_path"], + metrics_json=metrics, + feature_importance_json=result_data.get("feature_importance"), + confusion_matrix_json=result_data.get("confusion_matrix") or metrics.get("confusion_matrix"), + training_time_seconds=result_data.get("training_time_seconds", 0), + cross_val_scores=metrics.get("cv_scores"), + ) ) - - db.add(training_result) db.commit() return True - finally: db.close() - - except Exception as e: - print(f"Failed to save training result: {e}") + except Exception: return False @@ -329,42 +195,25 @@ def log_training_progress_sync( progress_percent: Optional[float] = None, epoch: Optional[int] = None, metrics: Optional[Dict[str, Any]] = None, - message: Optional[str] = None + message: Optional[str] = None, ) -> bool: - """ - Log training progress synchronously (for Celery tasks). - - Args: - job_id: Training job ID - progress_percent: Progress percentage (0-100) - epoch: Current epoch number - metrics: Current metrics - message: Progress message - - Returns: - True if logged successfully - """ + """Backward-compatible log helper for legacy workers.""" try: db_gen = get_sync_db() db = next(db_gen) - try: - # Create training log entry - log_entry = TrainingLog( - job_id=job_id, - progress_percent=progress_percent, - epoch=epoch, - metrics_json=metrics or {}, - message=message + db.add( + TrainingLog( + job_id=job_id, + progress_percent=progress_percent, + epoch=epoch, + metrics_json=metrics or {}, + message=message, + ) ) - - db.add(log_entry) db.commit() return True - finally: db.close() - - except Exception as e: - print(f"Failed to log training progress: {e}") - return False \ No newline at end of file + except Exception: + return False diff --git a/Backend/app/services/training_service_runs.py b/Backend/app/services/training_service_runs.py index d64461c..4c1c6da 100644 --- a/Backend/app/services/training_service_runs.py +++ b/Backend/app/services/training_service_runs.py @@ -1,141 +1,132 @@ -"""Training service layer - run-based architecture for ML training.""" -import uuid -from typing import Dict, Any, Optional +"""Training service layer for run-based ML training.""" from datetime import datetime, timezone +from typing import Any, Dict +import uuid + from fastapi import HTTPException, status +from sqlalchemy import and_, func, select from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy import select, func, and_, or_ -from app.models.training import TrainingRun -from app.models.experiment import Experiment from app.models.dataset import Dataset +from app.models.experiment import Experiment +from app.models.training import TrainingRun from app.services.task_manager import TaskManager +def _results_summary(run: TrainingRun): + if not run.results: + return None + return { + **run.results.get("summary", {}), + "best_model": run.results.get("best_model"), + } + + +def _progress_payload(run: TrainingRun) -> Dict[str, Any]: + if run.status == "completed": + return {"percent": 100, "message": "Training completed"} + if run.status == "failed": + return {"percent": 0, "message": run.error_message or "Training failed"} + if run.status == "cancelled": + return {"percent": 0, "message": "Training cancelled"} + if run.status == "running": + progress = 50 + message = "Training models..." + if isinstance(run.results, dict): + progress_data = run.results.get("progress") + if isinstance(progress_data, dict): + current = progress_data.get("current", 0) + total = progress_data.get("total", 0) + current_model = progress_data.get("current_model") + if isinstance(current, (int, float)) and isinstance(total, (int, float)) and total > 0: + progress = max(0, min(99, int((current / total) * 100))) + if current_model: + message = f"Training {current_model}..." + return {"percent": progress, "message": message} + return {"percent": 0, "message": "Waiting for worker..."} + + async def start_training_run( db: AsyncSession, experiment_id: uuid.UUID, - user_id: int + user_id: int, ) -> Dict[str, Any]: - """ - Start a new training run for entire config. - - Args: - db: Database session - experiment_id: Experiment ID - user_id: User ID - - Returns: - { - 'run_id': str, - 'run_number': int, - 'job_id': str, - 'status': 'pending', - 'created_at': str - } - - Raises: - HTTPException: If experiment not found or config invalid - """ - # 1. Verify experiment exists and belongs to user experiment_query = select(Experiment).where( and_(Experiment.id == experiment_id, Experiment.user_id == user_id) ) result = await db.execute(experiment_query) experiment = result.scalar_one_or_none() - + if not experiment: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Experiment not found" - ) - - # 2. Validate config completeness + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Experiment not found") + config = experiment.config - if not config or not config.get('taskType') or not config.get('targetColumn'): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Experiment config is incomplete. Please configure task type and target column." - ) - - if not config.get('selectedFeatures') or len(config.get('selectedFeatures', [])) == 0: + if not config or not config.get("taskType") or not config.get("targetColumn"): raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, - detail="No features selected for training" + detail="Experiment config is incomplete. Please configure task type and target column.", ) - - if not config.get('models') or len(config.get('models', [])) == 0: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="No models selected for training" - ) - - # 3. Get dataset - dataset_query = select(Dataset).where(Dataset.id == experiment.dataset_id) + + if not config.get("selectedFeatures"): + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="No features selected for training") + + if not config.get("models"): + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="No models selected for training") + + dataset_query = select(Dataset).where( + and_(Dataset.id == experiment.dataset_id, Dataset.user_id == user_id) + ) dataset_result = await db.execute(dataset_query) dataset = dataset_result.scalar_one_or_none() - + if not dataset: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Dataset not found" - ) - - # 4. Generate run_number atomically (prevent race condition) - # Lock the experiment row to prevent concurrent run number conflicts + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Dataset not found") + + # Lock the experiment row so two concurrent requests cannot allocate the same run number. lock_query = select(Experiment).where(Experiment.id == experiment_id).with_for_update() await db.execute(lock_query) - - # Get max run_number for this experiment + max_run_query = select(func.coalesce(func.max(TrainingRun.run_number), 0)).where( TrainingRun.experiment_id == experiment_id ) max_run_result = await db.execute(max_run_query) run_number = (max_run_result.scalar() or 0) + 1 - - # 5. Create training_run record with config snapshot + training_run = TrainingRun( experiment_id=experiment_id, run_number=run_number, - status='pending', - config_snapshot=config # Save immutable snapshot + status="pending", + config_snapshot=config, ) - db.add(training_run) await db.commit() await db.refresh(training_run) - - # 6. Start Celery task for entire run + task_manager = TaskManager() try: job_id = task_manager.start_training_run_task( run_id=str(training_run.id), experiment_id=str(experiment_id), - dataset_id=str(dataset.id) + dataset_id=str(dataset.id), ) - - # 7. Update run with job_id training_run.job_id = job_id await db.commit() - - except Exception as e: - # If Celery dispatch fails, mark run as failed - training_run.status = 'failed' - training_run.error_message = f"Failed to dispatch training task: {str(e)}" + except Exception as exc: + training_run.status = "failed" + training_run.error_message = f"Failed to dispatch training task: {exc}" training_run.completed_at = datetime.now(timezone.utc) await db.commit() - raise HTTPException( status_code=status.HTTP_503_SERVICE_UNAVAILABLE, - detail=f"Failed to start training: {str(e)}" - ) - + detail="Training service is temporarily unavailable", + ) from exc + return { - 'run_id': str(training_run.id), - 'run_number': run_number, - 'job_id': job_id, - 'status': 'pending', - 'created_at': training_run.created_at.isoformat() + "run_id": str(training_run.id), + "run_number": run_number, + "job_id": job_id, + "status": "pending", + "created_at": training_run.created_at.isoformat(), } @@ -144,42 +135,22 @@ async def get_experiment_runs( experiment_id: uuid.UUID, user_id: int, page: int = 1, - page_size: int = 20 + page_size: int = 20, ) -> Dict[str, Any]: - """ - Get paginated list of training runs for an experiment. - - Args: - db: Database session - experiment_id: Experiment ID - user_id: User ID - page: Page number (1-indexed) - page_size: Items per page - - Returns: - Paginated list of training runs - """ - # Verify ownership + page = max(1, page) + page_size = max(1, min(page_size, 100)) + experiment_query = select(Experiment).where( and_(Experiment.id == experiment_id, Experiment.user_id == user_id) ) result = await db.execute(experiment_query) - experiment = result.scalar_one_or_none() - - if not experiment: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Experiment not found" - ) - - # Get total count - count_query = select(func.count(TrainingRun.id)).where( - TrainingRun.experiment_id == experiment_id - ) + if not result.scalar_one_or_none(): + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Experiment not found") + + count_query = select(func.count(TrainingRun.id)).where(TrainingRun.experiment_id == experiment_id) count_result = await db.execute(count_query) total = count_result.scalar() or 0 - - # Get paginated runs + offset = (page - 1) * page_size runs_query = ( select(TrainingRun) @@ -190,134 +161,87 @@ async def get_experiment_runs( ) runs_result = await db.execute(runs_query) runs = runs_result.scalars().all() - + return { - 'runs': [ + "runs": [ { - 'id': str(run.id), - 'run_number': run.run_number, - 'status': run.status, - 'started_at': run.started_at.isoformat() if run.started_at else None, - 'completed_at': run.completed_at.isoformat() if run.completed_at else None, - 'duration_seconds': run.duration_seconds, - 'results_summary': { - **run.results.get('summary', {}), - 'best_model': run.results.get('best_model') - } if run.results else None, - 'created_at': run.created_at.isoformat() + "id": str(run.id), + "run_number": run.run_number, + "status": run.status, + "started_at": run.started_at.isoformat() if run.started_at else None, + "completed_at": run.completed_at.isoformat() if run.completed_at else None, + "duration_seconds": run.duration_seconds, + "progress": _progress_payload(run), + "results_summary": _results_summary(run), + "error_message": run.error_message, + "created_at": run.created_at.isoformat(), } for run in runs ], - 'total': total, - 'page': page, - 'page_size': page_size, - 'total_pages': (total + page_size - 1) // page_size if total > 0 else 0 + "total": total, + "page": page, + "page_size": page_size, + "total_pages": (total + page_size - 1) // page_size if total else 0, } async def get_run_details( db: AsyncSession, run_id: uuid.UUID, - user_id: int + user_id: int, ) -> Dict[str, Any]: - """ - Get complete details of a training run including all results. - - Args: - db: Database session - run_id: Training run ID - user_id: User ID - - Returns: - Complete run details with results - """ - # Get run with experiment to verify ownership run_query = ( select(TrainingRun) .join(Experiment, TrainingRun.experiment_id == Experiment.id) - .where( - and_( - TrainingRun.id == run_id, - Experiment.user_id == user_id - ) - ) + .where(and_(TrainingRun.id == run_id, Experiment.user_id == user_id)) ) result = await db.execute(run_query) run = result.scalar_one_or_none() - + if not run: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Training run not found" - ) - + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Training run not found") + return { - 'id': str(run.id), - 'experiment_id': str(run.experiment_id), - 'run_number': run.run_number, - 'status': run.status, - 'config_snapshot': run.config_snapshot, - 'results': run.results or {}, - 'artifacts': run.artifacts or {}, - 'started_at': run.started_at.isoformat() if run.started_at else None, - 'completed_at': run.completed_at.isoformat() if run.completed_at else None, - 'duration_seconds': run.duration_seconds, - 'error_message': run.error_message, - 'created_at': run.created_at.isoformat() + "id": str(run.id), + "experiment_id": str(run.experiment_id), + "run_number": run.run_number, + "status": run.status, + "progress": _progress_payload(run), + "config_snapshot": run.config_snapshot, + "results": run.results or {}, + "results_summary": _results_summary(run), + "artifacts": run.artifacts or {}, + "started_at": run.started_at.isoformat() if run.started_at else None, + "completed_at": run.completed_at.isoformat() if run.completed_at else None, + "duration_seconds": run.duration_seconds, + "error_message": run.error_message, + "created_at": run.created_at.isoformat(), } async def get_run_status( db: AsyncSession, run_id: uuid.UUID, - user_id: int + user_id: int, ) -> Dict[str, Any]: - """ - Get lightweight status of a training run (for polling). - - Args: - db: Database session - run_id: Training run ID - user_id: User ID - - Returns: - Run status info - """ - # Get run with experiment to verify ownership run_query = ( select(TrainingRun) .join(Experiment, TrainingRun.experiment_id == Experiment.id) - .where( - and_( - TrainingRun.id == run_id, - Experiment.user_id == user_id - ) - ) + .where(and_(TrainingRun.id == run_id, Experiment.user_id == user_id)) ) result = await db.execute(run_query) run = result.scalar_one_or_none() - + if not run: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Training run not found" - ) - - # Calculate progress - progress = 0 - if run.status == 'completed': - progress = 100 - elif run.status == 'running': - # Could enhance with real-time progress from Celery - progress = 50 - elif run.status == 'pending': - progress = 0 - + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Training run not found") + return { - 'run_id': str(run.id), - 'run_number': run.run_number, - 'status': run.status, - 'progress': progress, - 'error_message': run.error_message, - 'started_at': run.started_at.isoformat() if run.started_at else None + "run_id": str(run.id), + "run_number": run.run_number, + "status": run.status, + "progress": _progress_payload(run), + "results_summary": _results_summary(run), + "error_message": run.error_message, + "started_at": run.started_at.isoformat() if run.started_at else None, + "completed_at": run.completed_at.isoformat() if run.completed_at else None, } diff --git a/Backend/app/services/workspace_dataset_service.py b/Backend/app/services/workspace_dataset_service.py new file mode 100644 index 0000000..0a9d3c6 --- /dev/null +++ b/Backend/app/services/workspace_dataset_service.py @@ -0,0 +1,253 @@ +"""Database-free dataset operations scoped to a temporary guest session.""" +from __future__ import annotations + +import asyncio +import json +import threading +import time +import uuid +from pathlib import Path +from typing import Any + +import pandas as pd +from fastapi import HTTPException, UploadFile, status + +from app.services.dataset_service import ( + ALLOWED_EXTENSIONS, + MAX_FILE_SIZE, + MAX_PREVIEW_ROWS, + extract_file_metadata, + read_dataframe, + save_upload_file, +) +from app.services.session_manager import SessionExpired, SessionNotFound, session_manager + + +MANIFEST_FILE = "workspace.json" +_MANIFEST_LOCK = threading.RLock() + + +def _manifest_path(token: str) -> Path: + return session_manager.safe_path(token, MANIFEST_FILE) + + +def _empty_manifest() -> dict[str, Any]: + return { + "version": 1, + "updated_at": int(time.time()), + "datasets": {}, + "active_dataset_id": None, + } + + +def _load_manifest(token: str) -> dict[str, Any]: + path = _manifest_path(token) + if not path.exists(): + return _empty_manifest() + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail={ + "code": "WORKSPACE_MANIFEST_ERROR", + "message": "The temporary workspace metadata could not be read.", + }, + ) from exc + if not isinstance(payload, dict) or not isinstance(payload.get("datasets"), dict): + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail={ + "code": "WORKSPACE_MANIFEST_ERROR", + "message": "The temporary workspace metadata is invalid.", + }, + ) + return payload + + +def _save_manifest(token: str, manifest: dict[str, Any]) -> None: + path = _manifest_path(token) + manifest["updated_at"] = int(time.time()) + temp_path = path.with_suffix(".tmp") + temp_path.write_text(json.dumps(manifest, separators=(",", ":")), encoding="utf-8") + temp_path.replace(path) + + +def _dataset_from_manifest(token: str, dataset_id: str) -> tuple[dict[str, Any], dict[str, Any]]: + manifest = _load_manifest(token) + dataset = manifest["datasets"].get(dataset_id) + if not dataset: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail={"code": "DATASET_NOT_FOUND", "message": "This dataset is not part of the current session."}, + ) + return manifest, dataset + + +def _dataset_path(token: str, dataset: dict[str, Any]) -> Path: + return session_manager.safe_path(token, "datasets", dataset["stored_filename"]) + + +async def create_workspace_dataset( + token: str, + file: UploadFile, + name: str | None = None, + description: str | None = None, +) -> dict[str, Any]: + original_filename = Path(file.filename or "dataset").name + file_ext = Path(original_filename).suffix.lower() + if file_ext not in ALLOWED_EXTENSIONS: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={ + "code": "UNSUPPORTED_FILE_TYPE", + "message": f"Unsupported file type. Use one of: {', '.join(sorted(ALLOWED_EXTENSIONS))}.", + }, + ) + + clean_name = (name or Path(original_filename).stem or "Dataset").strip() + if not clean_name: + clean_name = "Dataset" + + dataset_id = str(uuid.uuid4()) + stored_filename = f"{dataset_id}{file_ext}" + + try: + destination = session_manager.safe_path(token, "datasets", stored_filename) + except (SessionExpired, SessionNotFound) as exc: + raise HTTPException( + status_code=status.HTTP_410_GONE, + detail={"code": "SESSION_EXPIRED", "message": "This temporary session has ended."}, + ) from exc + + try: + file_size = await save_upload_file(file, destination, MAX_FILE_SIZE) + metadata = await asyncio.to_thread(extract_file_metadata, str(destination)) + if metadata["row_count"] <= 0: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={"code": "EMPTY_DATASET", "message": "The uploaded dataset contains no data rows."}, + ) + except HTTPException: + destination.unlink(missing_ok=True) + raise + except (ValueError, pd.errors.ParserError, UnicodeDecodeError) as exc: + destination.unlink(missing_ok=True) + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={ + "code": "DATASET_PARSE_ERROR", + "message": "The dataset could not be parsed. Check that the file is valid and not corrupted.", + }, + ) from exc + except Exception as exc: + destination.unlink(missing_ok=True) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail={"code": "DATASET_PROCESSING_ERROR", "message": "The dataset could not be processed."}, + ) from exc + finally: + await file.close() + + dataset = { + "id": dataset_id, + "name": clean_name, + "description": description.strip() if description else None, + "original_filename": original_filename, + "stored_filename": stored_filename, + "file_size_bytes": file_size, + "row_count": metadata["row_count"], + "column_count": metadata["column_count"], + "column_info": metadata["column_info"], + "created_at": int(time.time()), + "temporary": True, + } + + try: + with _MANIFEST_LOCK: + manifest = _load_manifest(token) + manifest["datasets"][dataset_id] = dataset + manifest["active_dataset_id"] = dataset_id + _save_manifest(token, manifest) + except Exception: + destination.unlink(missing_ok=True) + raise + + return dataset.copy() + + +def list_workspace_datasets(token: str) -> list[dict[str, Any]]: + with _MANIFEST_LOCK: + manifest = _load_manifest(token) + datasets = list(manifest["datasets"].values()) + return sorted(datasets, key=lambda item: item.get("created_at", 0), reverse=True) + + +def get_workspace_dataset(token: str, dataset_id: str) -> dict[str, Any]: + with _MANIFEST_LOCK: + _, dataset = _dataset_from_manifest(token, dataset_id) + return dataset.copy() + + +def update_workspace_dataset( + token: str, + dataset_id: str, + name: str, + description: str | None = None, +) -> dict[str, Any]: + clean_name = name.strip() + if not clean_name: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={"code": "DATASET_NAME_REQUIRED", "message": "Dataset name cannot be empty."}, + ) + + with _MANIFEST_LOCK: + manifest, dataset = _dataset_from_manifest(token, dataset_id) + dataset["name"] = clean_name + if description is not None: + dataset["description"] = description.strip() or None + manifest["datasets"][dataset_id] = dataset + _save_manifest(token, manifest) + return dataset.copy() + + +def preview_workspace_dataset(token: str, dataset_id: str, rows: int = 10) -> dict[str, Any]: + with _MANIFEST_LOCK: + _, dataset = _dataset_from_manifest(token, dataset_id) + dataset = dataset.copy() + rows = max(1, min(rows, MAX_PREVIEW_ROWS)) + path = _dataset_path(token, dataset) + if not path.is_file(): + raise HTTPException( + status_code=status.HTTP_410_GONE, + detail={"code": "DATASET_EXPIRED", "message": "The temporary dataset file is no longer available."}, + ) + + try: + df = read_dataframe(path, nrows=rows) + except Exception as exc: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail={"code": "DATASET_PREVIEW_ERROR", "message": "The dataset preview could not be generated."}, + ) from exc + + safe_df = df.astype(object).where(pd.notna(df), None) + return { + "columns": [str(column) for column in df.columns], + "data": safe_df.values.tolist(), + "row_count": dataset["row_count"], + "preview_rows": len(df), + } + + +def delete_workspace_dataset(token: str, dataset_id: str) -> bool: + with _MANIFEST_LOCK: + manifest, dataset = _dataset_from_manifest(token, dataset_id) + path = _dataset_path(token, dataset) + path.unlink(missing_ok=True) + del manifest["datasets"][dataset_id] + if manifest.get("active_dataset_id") == dataset_id: + manifest["active_dataset_id"] = next(iter(manifest["datasets"]), None) + _save_manifest(token, manifest) + return True diff --git a/Backend/app/services/workspace_eda_service.py b/Backend/app/services/workspace_eda_service.py new file mode 100644 index 0000000..ca551c8 --- /dev/null +++ b/Backend/app/services/workspace_eda_service.py @@ -0,0 +1,122 @@ +"""EDA and visualization for temporary guest-session datasets.""" +from __future__ import annotations + +import asyncio +from typing import Any + +import pandas as pd +from fastapi import HTTPException, status + +from app.services.dataset_service import read_dataframe +from app.services.eda_service import ( + _bar, + _box, + _correlation, + _histogram, + _require_column, + _scatter, + categorize_columns, + compute_correlations, + compute_missing_data_summary, + compute_statistics, + detect_id_columns, + get_column_info, + get_preview_data, + sample_dataframe, +) +from app.services.session_manager import session_manager +from app.services.workspace_dataset_service import get_workspace_dataset + + +async def load_workspace_dataset(token: str, dataset_id: str) -> tuple[pd.DataFrame, dict[str, Any]]: + dataset = get_workspace_dataset(token, dataset_id) + path = session_manager.safe_path(token, "datasets", dataset["stored_filename"]) + if not path.is_file(): + raise HTTPException( + status_code=status.HTTP_410_GONE, + detail={"code": "DATASET_EXPIRED", "message": "The temporary dataset file is no longer available."}, + ) + + try: + df = await asyncio.to_thread(read_dataframe, path) + except Exception as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={"code": "DATASET_READ_ERROR", "message": "NoCodeML could not read this temporary dataset."}, + ) from exc + return df, dataset + + +async def get_workspace_eda_summary(token: str, dataset_id: str) -> dict[str, Any]: + df, dataset = await load_workspace_dataset(token, dataset_id) + id_columns = detect_id_columns(df) + numeric_columns, categorical_columns = categorize_columns(df) + + return { + "dataset_info": { + "id": dataset["id"], + "name": dataset["name"], + "row_count": len(df), + "column_count": len(df.columns), + "file_size_bytes": dataset["file_size_bytes"], + "file_name": dataset["original_filename"], + "memory_usage_bytes": int(df.memory_usage(deep=True).sum()), + "temporary": True, + }, + "columns": get_column_info(df, id_columns), + "numeric_columns": numeric_columns, + "categorical_columns": categorical_columns, + "id_columns": id_columns, + "statistics": compute_statistics(df, numeric_columns), + "correlations": compute_correlations(df, numeric_columns), + "missing_data_summary": compute_missing_data_summary(df), + "preview_data": get_preview_data(df, 100), + } + + +async def generate_workspace_plot_data( + token: str, + dataset_id: str, + plot_type: str, + x_column: str, + y_column: str | None, + group_by: str | None, +) -> dict[str, Any]: + df, _dataset = await load_workspace_dataset(token, dataset_id) + sampled, is_sampled, total_rows, displayed_rows = sample_dataframe(df) + kind = plot_type.strip().lower() + + if kind == "histogram": + x = _require_column(sampled, x_column, numeric=True) + data, layout = _histogram(sampled, x) + elif kind == "scatter": + x = _require_column(sampled, x_column, numeric=True) + y = _require_column(sampled, y_column, numeric=True) + group = _require_column(sampled, group_by) if group_by else None + data, layout = _scatter(sampled, x, y, group) + elif kind == "box": + x = _require_column(sampled, x_column, numeric=True) + group = _require_column(sampled, group_by) if group_by else None + data, layout = _box(sampled, x, group) + elif kind == "correlation": + data, layout = _correlation(sampled) + elif kind == "bar": + x = _require_column(sampled, x_column) + data, layout = _bar(sampled, x) + else: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={ + "code": "PLOT_TYPE_INVALID", + "message": "Choose histogram, scatter, box, correlation or bar.", + }, + ) + + return { + "data": data, + "layout": layout, + "is_sampled": is_sampled, + "total_rows": total_rows, + "displayed_rows": displayed_rows, + "plot_type": kind, + } diff --git a/Backend/app/services/workspace_export_service.py b/Backend/app/services/workspace_export_service.py new file mode 100644 index 0000000..6ff5f10 --- /dev/null +++ b/Backend/app/services/workspace_export_service.py @@ -0,0 +1,238 @@ +"""Downloadable exports for a temporary NoCodeML guest workspace.""" +from __future__ import annotations + +import json +import shutil +import zipfile +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import pandas as pd +from fastapi import HTTPException, status + +from app.services.download_naming import artifact_filename, slugify +from app.services.session_manager import session_manager +from app.services.workspace_dataset_service import get_workspace_dataset, list_workspace_datasets +from app.services.workspace_eda_service import get_workspace_eda_summary +from app.services.workspace_prediction_service import list_predictions, prediction_download +from app.services.workspace_training_service import workspace_training_runner + + +JSON_MEDIA = "application/json" +CSV_MEDIA = "text/csv" +ZIP_MEDIA = "application/zip" +MODEL_MEDIA = "application/octet-stream" + + +def _write_json(path: Path, payload: Any) -> None: + path.write_text(json.dumps(payload, indent=2, ensure_ascii=False, default=str), encoding="utf-8") + + +def _export_path(token: str, filename: str) -> Path: + return session_manager.safe_path(token, "exports", filename) + + +async def export_eda(token: str, dataset_id: str, kind: str) -> tuple[Path, str, str]: + dataset = get_workspace_dataset(token, dataset_id) + summary = await get_workspace_eda_summary(token, dataset_id) + kind = kind.strip().lower() + + if kind == "summary": + filename = artifact_filename(dataset["name"], "eda-summary", "json") + path = _export_path(token, filename) + _write_json(path, summary) + return path, filename, JSON_MEDIA + + if kind == "statistics": + filename = artifact_filename(dataset["name"], "statistics", "csv") + path = _export_path(token, filename) + statistics = summary.get("statistics") or {} + if statistics: + frame = pd.DataFrame.from_dict(statistics, orient="index") + frame.index.name = "column" + frame.reset_index().to_csv(path, index=False) + else: + pd.DataFrame(columns=["column"]).to_csv(path, index=False) + return path, filename, CSV_MEDIA + + if kind == "missing-values": + filename = artifact_filename(dataset["name"], "missing-values", "csv") + path = _export_path(token, filename) + rows = (summary.get("missing_data_summary") or {}).get("columns_with_missing") or [] + pd.DataFrame(rows, columns=["column", "missing_count", "missing_percent"]).to_csv(path, index=False) + return path, filename, CSV_MEDIA + + if kind == "correlations": + filename = artifact_filename(dataset["name"], "correlation-matrix", "csv") + path = _export_path(token, filename) + correlations = summary.get("correlations") + if correlations and correlations.get("columns") and correlations.get("matrix"): + columns = correlations["columns"] + frame = pd.DataFrame(correlations["matrix"], index=columns, columns=columns) + frame.index.name = "column" + frame.reset_index().to_csv(path, index=False) + else: + pd.DataFrame(columns=["column"]).to_csv(path, index=False) + return path, filename, CSV_MEDIA + + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={ + "code": "EDA_EXPORT_INVALID", + "message": "Choose summary, statistics, missing-values or correlations.", + }, + ) + + +def export_training(token: str, run_id: str, kind: str) -> tuple[Path, str, str]: + run = workspace_training_runner.get(token, run_id) + if run.get("status") not in {"completed", "failed"}: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail={"code": "TRAINING_NOT_FINISHED", "message": "Wait for the training run to finish before exporting it."}, + ) + dataset_name = str(run.get("dataset_name") or "dataset") + kind = kind.strip().lower() + + if kind == "summary": + filename = artifact_filename(dataset_name, "training-summary", "json") + path = _export_path(token, filename) + _write_json(path, run) + return path, filename, JSON_MEDIA + + if kind == "model-comparison": + filename = artifact_filename(dataset_name, "model-comparison", "csv") + path = _export_path(token, filename) + rows: list[dict[str, Any]] = [] + for result in run.get("results") or []: + row: dict[str, Any] = { + "model": result.get("model_type"), + "success": result.get("success"), + "training_time_seconds": result.get("training_time_seconds"), + "error": result.get("error"), + } + for metric, value in ((result.get("metrics") or {}).get("test") or {}).items(): + if isinstance(value, (str, int, float, bool)) or value is None: + row[f"test_{metric}"] = value + for metric, value in ((result.get("metrics") or {}).get("train") or {}).items(): + if isinstance(value, (str, int, float, bool)) or value is None: + row[f"train_{metric}"] = value + rows.append(row) + pd.DataFrame(rows).to_csv(path, index=False) + return path, filename, CSV_MEDIA + + if kind == "feature-importance": + filename = artifact_filename(dataset_name, "feature-importance", "csv") + path = _export_path(token, filename) + importance = (run.get("best_model") or {}).get("feature_importance") or {} + features = importance.get("features") or [] + values = importance.get("importance") or [] + pd.DataFrame({"feature": features, "importance": values}).to_csv(path, index=False) + return path, filename, CSV_MEDIA + + if kind == "best-model": + best = run.get("best_model") or {} + model_file = best.get("model_file") + if not model_file: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail={"code": "BEST_MODEL_MISSING", "message": "This run does not have a successful best-model artifact."}, + ) + source = session_manager.safe_path(token, "models", run_id, str(model_file)) + if not source.is_file(): + raise HTTPException( + status_code=status.HTTP_410_GONE, + detail={"code": "MODEL_EXPIRED", "message": "The temporary best-model file is no longer available."}, + ) + model_type = slugify(str(best.get("model_type") or "model"), fallback="model") + filename = artifact_filename(dataset_name, f"best-model-{model_type}", "joblib") + return source, filename, MODEL_MEDIA + + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={ + "code": "TRAINING_EXPORT_INVALID", + "message": "Choose summary, model-comparison, feature-importance or best-model.", + }, + ) + + +async def build_session_bundle(token: str) -> tuple[Path, str]: + datasets = list_workspace_datasets(token) + runs = workspace_training_runner.list(token) + predictions = list_predictions(token) + primary_name = datasets[0]["name"] if len(datasets) == 1 else "workspace" + now = datetime.now(timezone.utc) + filename = artifact_filename(primary_name, "session", "zip", timestamp=now) + bundle_path = _export_path(token, filename) + + with zipfile.ZipFile(bundle_path, "w", compression=zipfile.ZIP_DEFLATED) as archive: + readme = ( + "NoCodeML temporary session export\n" + "================================\n\n" + "This bundle was generated from a temporary guest session.\n" + "NoCodeML does not require this workspace to remain on the server after you leave.\n\n" + f"Datasets: {len(datasets)}\nTraining runs: {len(runs)}\nBatch prediction files: {len(predictions)}\n" + ) + archive.writestr("README.txt", readme) + + manifest = { + "exported_at": now.isoformat(), + "temporary_session": True, + "datasets": [ + {key: value for key, value in dataset.items() if key != "stored_filename"} + for dataset in datasets + ], + "training_runs": runs, + "predictions": predictions, + } + archive.writestr("session-summary.json", json.dumps(manifest, indent=2, ensure_ascii=False, default=str)) + + for dataset in datasets: + source = session_manager.safe_path(token, "datasets", dataset["stored_filename"]) + if source.is_file(): + original_ext = source.suffix.lower() or ".dat" + friendly = f"nocodeml_{slugify(dataset['name'])}_source{original_ext}" + archive.write(source, f"datasets/{friendly}") + + summary = await get_workspace_eda_summary(token, dataset["id"]) + ds_slug = slugify(dataset["name"]) + archive.writestr( + f"analysis/{ds_slug}/eda-summary.json", + json.dumps(summary, indent=2, ensure_ascii=False, default=str), + ) + stats = summary.get("statistics") or {} + if stats: + frame = pd.DataFrame.from_dict(stats, orient="index") + frame.index.name = "column" + archive.writestr(f"analysis/{ds_slug}/statistics.csv", frame.reset_index().to_csv(index=False)) + missing = (summary.get("missing_data_summary") or {}).get("columns_with_missing") or [] + archive.writestr( + f"analysis/{ds_slug}/missing-values.csv", + pd.DataFrame(missing, columns=["column", "missing_count", "missing_percent"]).to_csv(index=False), + ) + + for run in runs: + dataset_slug = slugify(str(run.get("dataset_name") or "dataset")) + model_slug = slugify(str((run.get("best_model") or {}).get("model_type") or "training"), fallback="training") + archive.writestr( + f"training/{dataset_slug}_{model_slug}_summary.json", + json.dumps(run, indent=2, ensure_ascii=False, default=str), + ) + if run.get("status") == "completed": + best = run.get("best_model") or {} + model_file = best.get("model_file") + if model_file: + model_path = session_manager.safe_path(token, "models", run["id"], str(model_file)) + if model_path.is_file(): + archive.write(model_path, f"models/{dataset_slug}_best-model_{model_slug}.joblib") + + for prediction in predictions: + try: + source, friendly = prediction_download(token, prediction["id"]) + except HTTPException: + continue + archive.write(source, f"predictions/{friendly}") + + return bundle_path, filename diff --git a/Backend/app/services/workspace_prediction_service.py b/Backend/app/services/workspace_prediction_service.py new file mode 100644 index 0000000..2fbe209 --- /dev/null +++ b/Backend/app/services/workspace_prediction_service.py @@ -0,0 +1,274 @@ +"""Predictions from temporary guest-session model artifacts.""" +from __future__ import annotations + +import io +import json +import time +import uuid +from pathlib import Path +from typing import Any + +import joblib +import numpy as np +import pandas as pd +from fastapi import HTTPException, UploadFile, status + +from app.services.download_naming import artifact_filename +from app.services.session_manager import session_manager +from app.services.workspace_training_service import workspace_training_runner + + +MAX_BATCH_FILE_SIZE = 100 * 1024 * 1024 + + +def _decode_predictions(predictions: Any, label_encoder: Any | None) -> np.ndarray: + values = np.asarray(predictions) + if label_encoder is None: + return values + return np.asarray(label_encoder.inverse_transform(values.astype(int))) + + +def _probability_labels(model: Any, label_encoder: Any | None) -> list[str]: + estimator = model.named_steps.get("model") if hasattr(model, "named_steps") else model + classes = np.asarray(getattr(estimator, "classes_", [])) + if classes.size == 0: + return [] + if label_encoder is not None: + try: + return [str(value) for value in label_encoder.inverse_transform(classes.astype(int))] + except Exception: + pass + return [str(value) for value in classes] + + +def _load_model(token: str, run_id: str) -> tuple[dict[str, Any], dict[str, Any]]: + run = workspace_training_runner.get(token, run_id) + if run.get("status") != "completed": + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail={"code": "MODEL_NOT_READY", "message": "Finish a successful training run before making predictions."}, + ) + + best = run.get("best_model") or {} + model_file = best.get("model_file") + if not model_file: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail={"code": "BEST_MODEL_MISSING", "message": "The completed run does not have a usable best model."}, + ) + + model_path = session_manager.safe_path(token, "models", run_id, str(model_file)) + if not model_path.is_file(): + raise HTTPException( + status_code=status.HTTP_410_GONE, + detail={"code": "MODEL_EXPIRED", "message": "The temporary trained model is no longer available."}, + ) + + try: + model_data = joblib.load(model_path) + except Exception as exc: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail={"code": "MODEL_LOAD_ERROR", "message": "The temporary trained model could not be loaded."}, + ) from exc + if not isinstance(model_data, dict) or "model" not in model_data: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail={"code": "MODEL_ARTIFACT_INVALID", "message": "The temporary trained model artifact is invalid."}, + ) + return run, model_data + + +def predict_single(token: str, run_id: str, features: dict[str, Any]) -> dict[str, Any]: + run, model_data = _load_model(token, run_id) + model = model_data["model"] + label_encoder = model_data.get("label_encoder") + required = [str(value) for value in (model_data.get("feature_columns") or [])] + if not required: + required = list(features.keys()) + + missing = [column for column in required if column not in features] + if missing: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={ + "code": "PREDICTION_FEATURES_MISSING", + "message": f"Provide the required feature(s): {', '.join(missing[:10])}.", + "missing_features": missing, + }, + ) + + frame = pd.DataFrame([{column: features[column] for column in required}]) + try: + raw_prediction = model.predict(frame) + except Exception as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={ + "code": "PREDICTION_VALUES_INVALID", + "message": "One or more feature values are incompatible with the trained model.", + }, + ) from exc + + decoded = _decode_predictions(raw_prediction, label_encoder) + prediction: Any = decoded[0] + if isinstance(prediction, np.generic): + prediction = prediction.item() + + probabilities = None + confidence = None + if hasattr(model, "predict_proba"): + try: + values = np.asarray(model.predict_proba(frame)[0], dtype=float) + labels = _probability_labels(model, label_encoder) + probabilities = { + labels[index] if index < len(labels) else str(index): float(probability) + for index, probability in enumerate(values) + } + confidence = float(values.max()) if values.size else None + except Exception: + probabilities = None + confidence = None + + return { + "prediction": prediction, + "probabilities": probabilities, + "confidence": confidence, + "model_type": (run.get("best_model") or {}).get("model_type"), + "run_id": run_id, + "temporary": True, + } + + +async def predict_batch(token: str, run_id: str, file: UploadFile) -> dict[str, Any]: + filename = Path(file.filename or "predictions.csv").name + if Path(filename).suffix.lower() != ".csv": + await file.close() + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={"code": "BATCH_FILE_TYPE", "message": "Batch prediction input must be a CSV file."}, + ) + + try: + content = await file.read(MAX_BATCH_FILE_SIZE + 1) + finally: + await file.close() + if len(content) > MAX_BATCH_FILE_SIZE: + raise HTTPException( + status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, + detail={"code": "BATCH_FILE_TOO_LARGE", "message": "Batch prediction CSV must be 100 MB or smaller."}, + ) + + try: + original = pd.read_csv(io.BytesIO(content)) + except Exception as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={"code": "BATCH_PARSE_ERROR", "message": "The batch prediction CSV could not be parsed."}, + ) from exc + if original.empty: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={"code": "BATCH_EMPTY", "message": "The batch prediction CSV contains no rows."}, + ) + + run, model_data = _load_model(token, run_id) + model = model_data["model"] + label_encoder = model_data.get("label_encoder") + required = [str(value) for value in (model_data.get("feature_columns") or [])] + if not required: + required = [str(column) for column in original.columns] + + missing = [column for column in required if column not in original.columns] + if missing: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={ + "code": "BATCH_FEATURES_MISSING", + "message": f"The CSV is missing required feature column(s): {', '.join(missing[:10])}.", + "missing_features": missing, + }, + ) + + features = original[required].copy() + try: + decoded = _decode_predictions(model.predict(features), label_encoder) + except Exception as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={"code": "BATCH_VALUES_INVALID", "message": "Some batch feature values are incompatible with the trained model."}, + ) from exc + + result = original.copy() + result["prediction"] = decoded + if hasattr(model, "predict_proba"): + try: + result["confidence"] = np.asarray(model.predict_proba(features), dtype=float).max(axis=1) + except Exception: + pass + + prediction_id = str(uuid.uuid4()) + dataset_name = str(run.get("dataset_name") or "dataset") + download_name = artifact_filename(dataset_name, "predictions", "csv") + stored_name = f"{prediction_id}.csv" + output_path = session_manager.safe_path(token, "predictions", stored_name) + result.to_csv(output_path, index=False) + + metadata = { + "id": prediction_id, + "run_id": run_id, + "dataset_id": run.get("dataset_id"), + "dataset_name": dataset_name, + "model_type": (run.get("best_model") or {}).get("model_type"), + "stored_filename": stored_name, + "download_filename": download_name, + "total_predictions": len(result), + "created_at": int(time.time()), + "temporary": True, + } + metadata_path = session_manager.safe_path(token, "predictions", f"{prediction_id}.json") + metadata_path.write_text(json.dumps(metadata, separators=(",", ":")), encoding="utf-8") + + public = {key: value for key, value in metadata.items() if key != "stored_filename"} + public["download_url"] = f"/api/v1/workspace/predictions/{prediction_id}/download" + return public + + +def list_predictions(token: str) -> list[dict[str, Any]]: + directory = session_manager.safe_path(token, "predictions") + results: list[dict[str, Any]] = [] + for path in directory.glob("*.json"): + try: + payload = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict) or not payload.get("id"): + continue + public = {key: value for key, value in payload.items() if key != "stored_filename"} + public["download_url"] = f"/api/v1/workspace/predictions/{payload['id']}/download" + results.append(public) + except (OSError, json.JSONDecodeError): + continue + return sorted(results, key=lambda item: item.get("created_at", 0), reverse=True) + + +def prediction_download(token: str, prediction_id: str) -> tuple[Path, str]: + metadata_path = session_manager.safe_path(token, "predictions", f"{prediction_id}.json") + if not metadata_path.is_file(): + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail={"code": "PREDICTION_NOT_FOUND", "message": "This prediction file is not part of the current session."}, + ) + try: + metadata = json.loads(metadata_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail={"code": "PREDICTION_METADATA_ERROR", "message": "Prediction metadata could not be read."}, + ) from exc + + file_path = session_manager.safe_path(token, "predictions", str(metadata.get("stored_filename") or "")) + if not file_path.is_file(): + raise HTTPException( + status_code=status.HTTP_410_GONE, + detail={"code": "PREDICTION_EXPIRED", "message": "The temporary prediction file is no longer available."}, + ) + return file_path, str(metadata.get("download_filename") or "nocodeml_predictions.csv") diff --git a/Backend/app/services/workspace_training_service.py b/Backend/app/services/workspace_training_service.py new file mode 100644 index 0000000..10ee8ab --- /dev/null +++ b/Backend/app/services/workspace_training_service.py @@ -0,0 +1,322 @@ +"""Bounded, database-free ML training for temporary guest workspaces.""" +from __future__ import annotations + +import json +import threading +import time +import uuid +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from typing import Any + +import joblib +from fastapi import HTTPException, status + +from app.core.config import settings +from app.services.model_trainer import ModelTrainer +from app.services.session_manager import SessionExpired, SessionNotFound, session_manager +from app.services.workspace_dataset_service import get_workspace_dataset + + +class WorkspaceModelTrainer(ModelTrainer): + """ModelTrainer variant that never writes to persistent artifact storage.""" + + def load_dataset(self, dataset_uri: str): + return self._read_dataframe(Path(dataset_uri)) + + def save_model(self, pipeline, model_id, label_encoder, feature_columns): + safe_id = "".join(character for character in model_id if character.isalnum() or character in {"-", "_"}) + safe_id = safe_id or str(uuid.uuid4()) + local_path = self.models_dir / f"{safe_id}.joblib" + artifact = { + "model": pipeline, + "label_encoder": label_encoder, + "feature_columns": feature_columns, + "saved_at": time.time(), + "artifact_version": 3, + } + joblib.dump(artifact, local_path) + return local_path.name + + +class WorkspaceTrainingRunner: + def __init__(self) -> None: + self._executor = ThreadPoolExecutor( + max_workers=settings.WORKSPACE_TRAINING_WORKERS, + thread_name_prefix="nocodeml-guest-training", + ) + self._lock = threading.RLock() + self._active_sessions: set[str] = set() + + @staticmethod + def _now() -> int: + return int(time.time()) + + def _run_path(self, token: str, run_id: str) -> Path: + return session_manager.safe_path(token, "training", f"{run_id}.json", touch=False) + + def _write_run(self, token: str, run_id: str, payload: dict[str, Any]) -> None: + path = self._run_path(token, run_id) + payload["updated_at"] = self._now() + temp = path.with_suffix(".tmp") + temp.write_text(json.dumps(payload, separators=(",", ":"), default=str), encoding="utf-8") + temp.replace(path) + + def _read_run(self, token: str, run_id: str) -> dict[str, Any]: + path = self._run_path(token, run_id) + if not path.is_file(): + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail={"code": "TRAINING_RUN_NOT_FOUND", "message": "This training run is not part of the current session."}, + ) + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail={"code": "TRAINING_STATUS_ERROR", "message": "Training status could not be read."}, + ) from exc + return payload + + def submit(self, token: str, config: dict[str, Any]) -> dict[str, Any]: + models = config.get("models") or [] + if not models: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={"code": "NO_MODELS_SELECTED", "message": "Choose at least one model to train."}, + ) + if len(models) > settings.WORKSPACE_MAX_MODELS_PER_RUN: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={ + "code": "TOO_MANY_MODELS", + "message": f"A run can train at most {settings.WORKSPACE_MAX_MODELS_PER_RUN} models.", + }, + ) + + dataset = get_workspace_dataset(token, config["dataset_id"]) + digest = session_manager.token_digest(token) + + with self._lock: + if digest in self._active_sessions: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail={ + "code": "TRAINING_ALREADY_RUNNING", + "message": "This session already has an active training run. Wait for it to finish before starting another.", + }, + ) + self._active_sessions.add(digest) + + run_id = str(uuid.uuid4()) + initial = { + "id": run_id, + "dataset_id": dataset["id"], + "dataset_name": dataset["name"], + "status": "queued", + "created_at": self._now(), + "progress": { + "current": 0, + "total": len(models), + "percent": 0, + "current_model": None, + "message": "Training is queued.", + }, + "config": { + "target_column": config["target_column"], + "task_type": config["task_type"], + "selected_features": config.get("selected_features"), + "models": [ + { + "model_type": model.get("model_type"), + "enable_optimization": bool(model.get("enable_optimization", False)), + "hyperparameters": model.get("hyperparameters") or {}, + } + for model in models + ], + "test_size": config.get("test_size", 0.2), + "random_state": config.get("random_state", 42), + "cv_folds": config.get("cv_folds", 3), + "scaling": config.get("scaling", True), + }, + "results": [], + "best_model": None, + "error": None, + "temporary": True, + } + + lease_acquired = False + try: + # Hold the session from the moment it enters the global queue. This + # prevents close/TTL cleanup from deleting a queued dataset before + # the worker thread starts executing the model fit. + session_manager.acquire_job(token) + lease_acquired = True + self._write_run(token, run_id, initial) + self._executor.submit(self._execute, token, run_id, config, digest) + except Exception: + if lease_acquired: + session_manager.release_job(token) + with self._lock: + self._active_sessions.discard(digest) + raise + + return initial.copy() + + def _execute(self, token: str, run_id: str, config: dict[str, Any], digest: str) -> None: + try: + dataset = get_workspace_dataset(token, config["dataset_id"]) + dataset_path = session_manager.safe_path(token, "datasets", dataset["stored_filename"], touch=False) + models_dir = session_manager.safe_path(token, "models", run_id, touch=False) + models_dir.mkdir(parents=True, exist_ok=True, mode=0o700) + trainer = WorkspaceModelTrainer(models_dir=str(models_dir)) + + run = self._read_run(token, run_id) + run["status"] = "running" + run["started_at"] = self._now() + self._write_run(token, run_id, run) + + model_specs = config["models"] + successful: list[dict[str, Any]] = [] + for index, model_spec in enumerate(model_specs, start=1): + session_manager.touch(token) + model_type = str(model_spec["model_type"]) + run = self._read_run(token, run_id) + run["progress"] = { + "current": index - 1, + "total": len(model_specs), + "percent": round(((index - 1) / len(model_specs)) * 100), + "current_model": model_type, + "message": f"Training {model_type}โ€ฆ", + } + self._write_run(token, run_id, run) + + result = trainer.train_complete_pipeline( + dataset_path=str(dataset_path), + target_column=config["target_column"], + model_type=model_type, + task_type=config["task_type"], + hyperparameters=model_spec.get("hyperparameters") or {}, + preprocessing_config={}, + training_config={ + "test_size": config.get("test_size", 0.2), + "random_state": config.get("random_state", 42), + "cv_folds": config.get("cv_folds", 3), + "scaling": config.get("scaling", True), + }, + selected_features=config.get("selected_features"), + job_id=f"{run_id}-{model_type}", + enable_optimization=bool(model_spec.get("enable_optimization", False)), + ) + + public_result = { + "model_type": model_type, + "success": bool(result.get("success")), + "metrics": result.get("metrics"), + "feature_importance": result.get("feature_importance"), + "confusion_matrix": result.get("confusion_matrix"), + "training_time_seconds": result.get("training_time_seconds"), + "dataset_info": result.get("dataset_info"), + "hyperparameters": result.get("hyperparameters"), + "model_file": result.get("model_path") if result.get("success") else None, + "error": result.get("error") if not result.get("success") else None, + } + run = self._read_run(token, run_id) + run["results"].append(public_result) + if public_result["success"]: + successful.append(public_result) + run["progress"] = { + "current": index, + "total": len(model_specs), + "percent": round((index / len(model_specs)) * 100), + "current_model": model_type, + "message": f"Finished {model_type}.", + } + self._write_run(token, run_id, run) + + run = self._read_run(token, run_id) + if successful: + run["best_model"] = self._choose_best(successful, config["task_type"]) + run["status"] = "completed" + run["progress"] = { + "current": len(model_specs), + "total": len(model_specs), + "percent": 100, + "current_model": None, + "message": "Training complete.", + } + else: + run["status"] = "failed" + run["error"] = { + "code": "ALL_MODELS_FAILED", + "message": "None of the selected models could be trained with this dataset and configuration.", + } + run["completed_at"] = self._now() + self._write_run(token, run_id, run) + except (SessionExpired, SessionNotFound): + return + except Exception as exc: + try: + run = self._read_run(token, run_id) + run["status"] = "failed" + run["error"] = { + "code": "TRAINING_RUN_FAILED", + "message": str(exc)[:500] or "Training failed unexpectedly.", + } + run["completed_at"] = self._now() + self._write_run(token, run_id, run) + except Exception: + pass + finally: + session_manager.release_job(token) + with self._lock: + self._active_sessions.discard(digest) + + @staticmethod + def _choose_best(results: list[dict[str, Any]], task_type: str) -> dict[str, Any]: + metric_name = "f1_score" if task_type == "classification" else "r2_score" + + def score(result: dict[str, Any]) -> float: + return float((result.get("metrics") or {}).get("test", {}).get(metric_name, float("-inf"))) + + best = max(results, key=score) + return { + "model_type": best["model_type"], + "model_file": best["model_file"], + "metric": metric_name, + "score": score(best), + "metrics": best.get("metrics"), + "feature_importance": best.get("feature_importance"), + "confusion_matrix": best.get("confusion_matrix"), + } + + def get(self, token: str, run_id: str) -> dict[str, Any]: + run = self._read_run(token, run_id) + digest = session_manager.token_digest(token) + if run.get("status") in {"queued", "running"}: + with self._lock: + active = digest in self._active_sessions + if not active: + run["status"] = "failed" + run["error"] = { + "code": "TRAINING_INTERRUPTED", + "message": "The training process restarted before this temporary run completed. Start the run again.", + } + run["completed_at"] = self._now() + self._write_run(token, run_id, run) + return run + + def list(self, token: str) -> list[dict[str, Any]]: + training_dir = session_manager.safe_path(token, "training") + runs: list[dict[str, Any]] = [] + for path in training_dir.glob("*.json"): + try: + payload = json.loads(path.read_text(encoding="utf-8")) + if isinstance(payload, dict) and payload.get("id"): + runs.append(payload) + except (OSError, json.JSONDecodeError): + continue + return sorted(runs, key=lambda item: item.get("created_at", 0), reverse=True) + + +workspace_training_runner = WorkspaceTrainingRunner() diff --git a/Backend/app/worker/celery_app.py b/Backend/app/worker/celery_app.py index 64d1d1e..537ad24 100644 --- a/Backend/app/worker/celery_app.py +++ b/Backend/app/worker/celery_app.py @@ -1,41 +1,36 @@ from celery import Celery + from app.core.config import settings + celery_app = Celery( "worker", broker=settings.CELERY_BROKER_URL, backend=settings.CELERY_RESULT_BACKEND, - include=["app.worker.tasks", "app.worker.training_tasks"], # Import both task modules - broker_connection_retry_on_startup=True # Retry connecting to Redis on startup + include=[ + "app.worker.tasks", + "app.worker.training_tasks", # V2 job compatibility + "app.worker.run_tasks", # active V3 run architecture + ], + broker_connection_retry_on_startup=True, ) -# Configure Celery for training tasks celery_app.conf.update( - # Use default 'celery' queue (no routing needed) - # Task execution settings - task_serializer='json', - accept_content=['json'], - result_serializer='json', - timezone='UTC', + task_serializer="json", + accept_content=["json"], + result_serializer="json", + timezone="UTC", enable_utc=True, - - # Task timeouts and retries for ML training - task_soft_time_limit=3300, # 55 minutes soft limit - task_time_limit=3600, # 1 hour hard limit - task_default_retry_delay=300, # 5 minutes between retries - task_max_retries=1, # Only retry once for training tasks - - # Result backend settings - result_expires=86400, # Results expire after 24 hours + task_soft_time_limit=3300, + task_time_limit=3600, + task_default_retry_delay=300, + task_max_retries=1, + result_expires=86400, task_track_started=True, task_send_sent_event=True, - - # Worker settings for ML training - worker_prefetch_multiplier=1, # One task at a time for training - worker_max_tasks_per_child=5, # Restart worker after 5 tasks to prevent memory leaks + worker_prefetch_multiplier=1, + worker_max_tasks_per_child=5, worker_disable_rate_limits=True, - - # Task acknowledgment task_acks_late=True, task_reject_on_worker_lost=True, -) \ No newline at end of file +) diff --git a/Backend/app/worker/run_tasks.py b/Backend/app/worker/run_tasks.py new file mode 100644 index 0000000..66eb1fd --- /dev/null +++ b/Backend/app/worker/run_tasks.py @@ -0,0 +1,283 @@ +"""V3 run-based Celery training tasks. + +The V3 worker consumes the exact experiment snapshot stored with a TrainingRun. +It intentionally lives separately from the V2 job worker so the legacy task +contract can remain untouched while the active release uses one consistent path. +""" +from __future__ import annotations + +import traceback +import uuid +from datetime import datetime, timezone +from typing import Any, Dict + +from app.core.model_defaults import DEFAULT_HYPERPARAMETERS +from app.db.sync_session import SyncSessionLocal +from app.models.dataset import Dataset +from app.models.training import TrainingRun +from app.services.model_trainer import ModelTrainer +from app.worker.celery_app import celery_app + + +def resolve_model_hyperparameters(model_cfg: Dict[str, Any], task_type: str, model_type: str) -> Dict[str, Any]: + """Resolve V3 parameters while retaining compatibility with old snapshots.""" + base = dict(DEFAULT_HYPERPARAMETERS.get(task_type, {}).get(model_type, {})) + + # Current V3 shape. + configured = model_cfg.get("hyperparameters") + custom = model_cfg.get("custom_hyperparameters") or model_cfg.get("customHyperparameters") + + # V2 snapshots sometimes nested the resolved values under config. + legacy_config = model_cfg.get("config") if isinstance(model_cfg.get("config"), dict) else {} + if configured is None: + configured = legacy_config.get("hyperparameters") + if custom is None: + custom = legacy_config.get("custom_hyperparameters") or legacy_config.get("customHyperparameters") + + if isinstance(configured, dict): + base.update(configured) + if isinstance(custom, dict): + base.update(custom) + return base + + +def training_config_from_snapshot(config: Dict[str, Any]) -> Dict[str, Any]: + """Convert the UI's train ratio into the trainer's test-size contract.""" + train_ratio = config.get("trainTestSplit", 0.8) + try: + train_ratio = float(train_ratio) + except (TypeError, ValueError): + train_ratio = 0.8 + train_ratio = max(0.6, min(0.9, train_ratio)) + + try: + random_seed = int(config.get("randomSeed", 42)) + except (TypeError, ValueError): + random_seed = 42 + + return { + "test_size": round(1.0 - train_ratio, 4), + "random_state": random_seed, + "cv_folds": 3, + "scaling": True, + } + + +def _best_model(model_results: list[Dict[str, Any]], task_type: str): + successful = [item for item in model_results if item.get("metrics", {}).get("test")] + if not successful: + return None, successful + + if task_type == "classification": + for metric in ("accuracy", "f1_score", "precision", "recall"): + candidates = [item for item in successful if item["metrics"]["test"].get(metric) is not None] + if candidates: + winner = max(candidates, key=lambda item: item["metrics"]["test"][metric]) + return { + "model_type": winner["model_type"], + "display_name": winner["display_name"], + "metric": metric, + "value": winner["metrics"]["test"][metric], + }, successful + else: + r2_candidates = [item for item in successful if item["metrics"]["test"].get("r2_score") is not None] + if r2_candidates: + winner = max(r2_candidates, key=lambda item: item["metrics"]["test"]["r2_score"]) + return { + "model_type": winner["model_type"], + "display_name": winner["display_name"], + "metric": "r2_score", + "value": winner["metrics"]["test"]["r2_score"], + }, successful + + mae_candidates = [item for item in successful if item["metrics"]["test"].get("mae") is not None] + if mae_candidates: + winner = min(mae_candidates, key=lambda item: item["metrics"]["test"]["mae"]) + return { + "model_type": winner["model_type"], + "display_name": winner["display_name"], + "metric": "mae", + "value": winner["metrics"]["test"]["mae"], + }, successful + + return None, successful + + +@celery_app.task(bind=True, name="app.worker.run_tasks.train_config_run_v3") +def train_config_run_v3(self, run_id: str, experiment_id: str, dataset_id: str): + """Train all models from an immutable V3 TrainingRun config snapshot.""" + del experiment_id # The run and dataset records are the source of truth here. + + db = SyncSessionLocal() + run_uuid = uuid.UUID(run_id) + training_run = None + + try: + training_run = db.query(TrainingRun).filter(TrainingRun.id == run_uuid).first() + if not training_run: + raise ValueError("Training run not found") + + dataset = db.query(Dataset).filter(Dataset.id == uuid.UUID(dataset_id)).first() + if not dataset: + raise ValueError("Dataset not found") + + config = dict(training_run.config_snapshot or {}) + task_type = config.get("taskType") + target_column = config.get("targetColumn") + selected_features = list(config.get("selectedFeatures") or []) + feature_types = config.get("featureTypes") or {} + model_configs = list(config.get("models") or []) + optimization_enabled = bool(config.get("enableOptimization", False)) + + if task_type not in {"classification", "regression"}: + raise ValueError("Training task type is missing or invalid") + if not target_column: + raise ValueError("Target column is missing") + if not selected_features: + raise ValueError("No training features were selected") + if not model_configs: + raise ValueError("No models were selected") + + training_run.status = "running" + training_run.started_at = datetime.now(timezone.utc) + training_run.error_message = None + db.commit() + + trainer = ModelTrainer() + trainer_config = training_config_from_snapshot(config) + total_models = len(model_configs) + model_results: list[Dict[str, Any]] = [] + dataset_info = None + + for index, model_cfg in enumerate(model_configs, start=1): + model_type = model_cfg.get("model_type") or model_cfg.get("modelType") + if not model_type: + model_results.append({"model_type": "unknown", "display_name": "Unknown model", "error": "Model type is missing"}) + continue + + display_name = model_cfg.get("display_name") or model_cfg.get("displayName") or model_type + training_run.results = { + "progress": { + "current": index, + "total": total_models, + "current_model": display_name, + } + } + db.commit() + + self.update_state( + state="PROGRESS", + meta={ + "current": index, + "total": total_models, + "status": f"Training {display_name}", + "run_id": run_id, + "run_number": training_run.run_number, + }, + ) + + try: + result = trainer.train_complete_pipeline( + dataset_path=dataset.storage_path, + target_column=target_column, + model_type=model_type, + task_type=task_type, + hyperparameters=resolve_model_hyperparameters(model_cfg, task_type, model_type), + training_config=trainer_config, + selected_features=selected_features, + feature_types=feature_types, + job_id=f"{run_id}_{model_type}", + enable_optimization=optimization_enabled, + ) + + if not result.get("success"): + raise ValueError(result.get("error") or "Training failed") + + dataset_info = dataset_info or result.get("dataset_info") + model_results.append( + { + "model_type": model_type, + "display_name": display_name, + "metrics": result.get("metrics", {}), + "feature_importance": result.get("feature_importance"), + "confusion_matrix": result.get("confusion_matrix"), + "model_path": result.get("model_path"), + "training_time": result.get("training_time_seconds", 0), + "hyperparameters": result.get("hyperparameters", {}), + "hyperparameter_tuning": result.get("hyperparameter_tuning"), + } + ) + except Exception as exc: + model_results.append( + { + "model_type": model_type, + "display_name": display_name, + "error": str(exc)[:500], + } + ) + + best_model, successful = _best_model(model_results, task_type) + summary = { + "total_models": total_models, + "successful": len(successful), + "failed": total_models - len(successful), + } + results = { + "task_type": task_type, + "dataset_info": dataset_info, + "models": model_results, + "best_model": best_model, + "summary": summary, + "training_config": { + "train_ratio": round(1 - trainer_config["test_size"], 4), + "test_ratio": trainer_config["test_size"], + "random_seed": trainer_config["random_state"], + }, + } + + training_run.completed_at = datetime.now(timezone.utc) + training_run.duration_seconds = int((training_run.completed_at - training_run.started_at).total_seconds()) + training_run.results = results + training_run.artifacts = { + "models": { + item["model_type"]: item["model_path"] + for item in model_results + if item.get("model_path") + } + } + + if successful: + training_run.status = "completed" + training_run.error_message = None + else: + training_run.status = "failed" + training_run.error_message = "All selected models failed to train. Review the model errors in this run." + + db.commit() + + if not successful: + raise RuntimeError(training_run.error_message) + + return { + "status": "SUCCESS", + "run_id": run_id, + "run_number": training_run.run_number, + "results": results, + } + + except Exception as exc: + traceback.print_exc() + if training_run is not None: + try: + if training_run.status != "completed": + training_run.status = "failed" + training_run.completed_at = training_run.completed_at or datetime.now(timezone.utc) + if training_run.started_at: + training_run.duration_seconds = int((training_run.completed_at - training_run.started_at).total_seconds()) + training_run.error_message = training_run.error_message or str(exc)[:500] + db.commit() + except Exception: + db.rollback() + raise + finally: + db.close() diff --git a/Backend/docker-compose.production.yaml b/Backend/docker-compose.production.yaml new file mode 100644 index 0000000..a69ef48 --- /dev/null +++ b/Backend/docker-compose.production.yaml @@ -0,0 +1,101 @@ +name: nocodeml-v3 + +services: + api: + build: + context: . + restart: unless-stopped + env_file: + - .env.production + environment: + PYTHONPATH: /app + ENVIRONMENT: production + CELERY_BROKER_URL: redis://redis:6379/0 + CELERY_RESULT_BACKEND: redis://redis:6379/0 + ARTIFACT_STORAGE_BACKEND: local + DATASETS_DIR: /data/datasets + MODELS_DIR: /data/models + PREDICTIONS_DIR: /data/predictions + expose: + - "8000" + volumes: + - artifacts:/data + command: >- + sh -c "alembic upgrade head && + exec uvicorn app.main:app --host 0.0.0.0 --port 8000 --proxy-headers --forwarded-allow-ips='*'" + depends_on: + redis: + condition: service_healthy + healthcheck: + test: + - CMD + - python + - -c + - "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5).read()" + interval: 15s + timeout: 6s + retries: 10 + start_period: 45s + + worker: + build: + context: . + restart: unless-stopped + env_file: + - .env.production + environment: + PYTHONPATH: /app + ENVIRONMENT: production + CELERY_BROKER_URL: redis://redis:6379/0 + CELERY_RESULT_BACKEND: redis://redis:6379/0 + ARTIFACT_STORAGE_BACKEND: local + DATASETS_DIR: /data/datasets + MODELS_DIR: /data/models + PREDICTIONS_DIR: /data/predictions + volumes: + - artifacts:/data + command: celery -A app.worker.celery_app worker --loglevel=info --concurrency=1 + depends_on: + api: + condition: service_healthy + redis: + condition: service_healthy + + redis: + image: redis:7-alpine + restart: unless-stopped + command: redis-server --appendonly yes --maxmemory 256mb --maxmemory-policy allkeys-lru + volumes: + - redis_data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 3s + retries: 10 + + caddy: + image: caddy:2-alpine + restart: unless-stopped + env_file: + - .env.production + ports: + - "80:80" + - "443:443" + - "443:443/udp" + volumes: + - ./Caddyfile:/etc/caddy/Caddyfile:ro + - caddy_data:/data + - caddy_config:/config + depends_on: + api: + condition: service_healthy + +volumes: + artifacts: + name: nocodeml_v3_artifacts + redis_data: + name: nocodeml_v3_redis + caddy_data: + name: nocodeml_v3_caddy_data + caddy_config: + name: nocodeml_v3_caddy_config diff --git a/Backend/docker-compose.yaml b/Backend/docker-compose.yaml index c3ccf9b..a35bdf3 100644 --- a/Backend/docker-compose.yaml +++ b/Backend/docker-compose.yaml @@ -1,83 +1,23 @@ +name: nocodeml + services: - fastapi_app: + api: build: . - container_name: fastapi_app working_dir: /app ports: - "8000:8000" - volumes: - - ./app:/app/app:ro # Read-only for safety - - dataset_storage:/app/datasets - - models_storage:/app/models # Persistent storage for trained models - - predictions_storage:/app/predictions # Persistent storage for prediction results - env_file: - - .env - environment: - - PYTHONPATH=/app - command: uvicorn app.main:app --host 0.0.0.0 --port 8000 --reload - depends_on: - postgres: - condition: service_healthy - redis: - condition: service_started - - celery_worker: - build: . - container_name: celery_worker - working_dir: /app volumes: - ./app:/app/app:ro - - dataset_storage:/app/datasets - - models_storage:/app/models # Celery worker needs write access to save models - - predictions_storage:/app/predictions # Storage for prediction results + tmpfs: + - /tmp/nocodeml-sessions:mode=0700,uid=10001,gid=10001 + - /tmp/nocodeml-legacy:mode=0700,uid=10001,gid=10001 + - /tmp/nocodeml-artifacts:mode=0700,uid=10001,gid=10001 env_file: - .env environment: - - PYTHONPATH=/app - command: celery -A app.worker.celery_app worker --loglevel=info --concurrency=2 - depends_on: - postgres: - condition: service_healthy - redis: - condition: service_started - - postgres: - image: postgres:17-alpine # Use alpine for smaller image - container_name: postgres_db - env_file: - - .env - ports: - - "5433:5432" - volumes: - - postgres_data:/var/lib/postgresql/data - healthcheck: - test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"] - interval: 5s - timeout: 5s - retries: 5 - - redis: - image: redis:7-alpine - container_name: redis_broker - healthcheck: - test: ["CMD", "redis-cli", "ping"] - interval: 5s - timeout: 3s - retries: 5 - - cloudflared: - image: cloudflare/cloudflared:latest - container_name: cloudflared - command: tunnel --no-autoupdate run --token ${CLOUDFLARE_TUNNEL_TOKEN} - restart: unless-stopped - network_mode: "service:fastapi_app" - depends_on: - - fastapi_app - profiles: - - tunnel + PYTHONPATH: /app + SESSION_ROOT_DIR: /tmp/nocodeml-sessions + command: uvicorn app.main:app --host 0.0.0.0 --port 8000 --reload -volumes: - postgres_data: - dataset_storage: - models_storage: # Persistent storage for trained ML models - predictions_storage: # Persistent storage for prediction results +# No Postgres, Redis, Celery worker or persistent application volume is part of +# the V3 guest runtime. All visitor workspaces are short-lived filesystem data. diff --git a/Backend/requirements.txt b/Backend/requirements.txt index d9bd14d..1435e65 100644 --- a/Backend/requirements.txt +++ b/Backend/requirements.txt @@ -1,46 +1,49 @@ # Core Framework -fastapi -uvicorn[standard] -starlette +fastapi>=0.115,<1 +uvicorn[standard]>=0.30,<1 +starlette>=0.40,<1 +httpx>=0.27,<1 # Database -SQLAlchemy[asyncio] -psycopg[binary] -aiosqlite -alembic +SQLAlchemy[asyncio]>=2.0,<3 +psycopg[binary]>=3.2,<4 +aiosqlite>=0.20,<1 +alembic>=1.13,<2 # Configuration & Validation -pydantic -pydantic-settings -email-validator +pydantic>=2.9,<3 +pydantic-settings>=2.5,<3 +email-validator>=2.2,<3 # Authentication -python-jose[cryptography] -passlib[bcrypt] +python-jose[cryptography]>=3.3,<4 +passlib[bcrypt]>=1.7,<2 bcrypt<4.1 -python-multipart +python-multipart>=0.0.9,<1 # Task Queue -celery -redis -kombu -vine +celery>=5.4,<6 +redis>=5,<7 +kombu>=5.4,<6 +vine>=5.1,<6 # Data Processing -pandas -numpy -openpyxl -pyarrow -plotly +pandas>=2.2,<4 +numpy>=1.26,<3 +openpyxl>=3.1,<4 +pyarrow>=16,<25 +plotly>=5.24,<7 + +# Object Storage +boto3>=1.35,<2 # Machine Learning -scikit-learn -xgboost -lightgbm -joblib -imbalanced-learn -autoclean +scikit-learn>=1.5,<2 +xgboost>=2.1,<4 +lightgbm>=4.5,<5 +joblib>=1.4,<2 +imbalanced-learn>=0.12,<1 # Utilities -python-dateutil -pytz +python-dateutil>=2.9,<3 +pytz>=2024,<2027 diff --git a/Backend/tests/test_eda_identity_detection.py b/Backend/tests/test_eda_identity_detection.py new file mode 100644 index 0000000..4f9ad33 --- /dev/null +++ b/Backend/tests/test_eda_identity_detection.py @@ -0,0 +1,48 @@ +import pandas as pd + +from app.services.eda_service import detect_id_columns + + +def test_identifier_names_are_detected_without_substring_false_positives(): + frame = pd.DataFrame( + { + "customer_id": ["a", "b", "c", "d"], + "paid": [10, 20, 30, 40], + "humidity": [41.2, 43.8, 40.1, 45.7], + "price": [101.3, 205.7, 309.1, 412.4], + } + ) + + detected = detect_id_columns(frame) + + assert "customer_id" in detected + assert "paid" not in detected + assert "humidity" not in detected + assert "price" not in detected + + +def test_row_number_sequence_is_detected_but_arbitrary_unique_numeric_feature_is_not(): + frame = pd.DataFrame( + { + "row_number": [1, 2, 3, 4, 5], + "measurement": [11, 37, 82, 145, 233], + } + ) + + detected = detect_id_columns(frame) + + assert "row_number" in detected + assert "measurement" not in detected + + +def test_non_sequential_unique_strings_are_not_assumed_to_be_ids(): + frame = pd.DataFrame( + { + "city_code": ["BLR", "HYD", "MAA", "DEL"], + "target": [0, 1, 0, 1], + } + ) + + detected = detect_id_columns(frame) + + assert "city_code" not in detected diff --git a/Backend/tests/test_ml_pipeline.py b/Backend/tests/test_ml_pipeline.py new file mode 100644 index 0000000..95b8524 --- /dev/null +++ b/Backend/tests/test_ml_pipeline.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +import joblib +import pandas as pd + +from app.services.artifact_store import artifact_store +from app.services.model_trainer import ModelTrainer + + +def _classification_frame(rows: int = 80) -> pd.DataFrame: + records = [] + cities = ["Bengaluru", "Hyderabad", "Chennai"] + for index in range(rows): + age = 18 + (index % 35) + income = 25000 + (index * 1375) % 90000 + city = cities[index % len(cities)] + target = int((age >= 32) or city == "Bengaluru") + records.append({"age": age, "income": income, "city": city, "target": target}) + return pd.DataFrame(records) + + +def _regression_frame(rows: int = 90) -> pd.DataFrame: + records = [] + zones = ["north", "south", "central"] + for index in range(rows): + area = 500 + index * 13 + rooms = 1 + index % 5 + zone = zones[index % len(zones)] + zone_bonus = {"north": 12000, "south": 7000, "central": 18000}[zone] + price = 150000 + area * 210 + rooms * 9500 + zone_bonus + records.append({"area": area, "rooms": rooms, "zone": zone, "price": price}) + return pd.DataFrame(records) + + +def test_classification_pipeline_persists_preprocessing_and_accepts_numeric_labels(tmp_path): + dataset_path = tmp_path / "classification.csv" + _classification_frame().to_csv(dataset_path, index=False) + + trainer = ModelTrainer(models_dir=str(tmp_path / "models")) + result = trainer.train_complete_pipeline( + dataset_path=str(dataset_path), + target_column="target", + model_type="LogisticRegression", + task_type="classification", + hyperparameters={"max_iter": 500}, + training_config={"test_size": 0.2, "random_state": 42, "cv_folds": 3, "scaling": True}, + selected_features=["age", "income", "city"], + job_id="classification-smoke", + ) + + assert result["success"] is True, result + assert 0 <= result["metrics"]["test"]["accuracy"] <= 1 + + with artifact_store.materialize(result["model_path"]) as model_path: + artifact = joblib.load(model_path) + + assert artifact["artifact_version"] == 3 + assert artifact["feature_columns"] == ["age", "income", "city"] + assert artifact["label_encoder"] is not None + + pipeline = artifact["model"] + # "Mysuru" was not present during training. OneHotEncoder(handle_unknown="ignore") + # must still allow inference without rebuilding category mappings. + prediction = pipeline.predict(pd.DataFrame([{"age": 27, "income": 64000, "city": "Mysuru"}])) + assert len(prediction) == 1 + decoded = artifact["label_encoder"].inverse_transform(prediction.astype(int)) + assert decoded[0] in {"0", "1"} + + +def test_regression_pipeline_trains_and_reloads_with_mixed_features(tmp_path): + dataset_path = tmp_path / "regression.csv" + _regression_frame().to_csv(dataset_path, index=False) + + trainer = ModelTrainer(models_dir=str(tmp_path / "models")) + result = trainer.train_complete_pipeline( + dataset_path=str(dataset_path), + target_column="price", + model_type="LinearRegression", + task_type="regression", + hyperparameters={}, + training_config={"test_size": 0.2, "random_state": 42, "cv_folds": 3, "scaling": True}, + selected_features=["area", "rooms", "zone"], + job_id="regression-smoke", + ) + + assert result["success"] is True, result + assert "r2_score" in result["metrics"]["test"] + + with artifact_store.materialize(result["model_path"]) as model_path: + artifact = joblib.load(model_path) + + pipeline = artifact["model"] + prediction = pipeline.predict(pd.DataFrame([{"area": 1050, "rooms": 3, "zone": "east"}])) + assert len(prediction) == 1 + assert float(prediction[0]) > 0 diff --git a/Backend/tests/test_run_config.py b/Backend/tests/test_run_config.py new file mode 100644 index 0000000..7b63a95 --- /dev/null +++ b/Backend/tests/test_run_config.py @@ -0,0 +1,33 @@ +from app.worker.run_tasks import resolve_model_hyperparameters, training_config_from_snapshot + + +def test_v3_hyperparameters_use_top_level_values_and_custom_overrides(): + config = { + "model_type": "RandomForestClassifier", + "hyperparameters": {"n_estimators": 111, "max_depth": 9}, + "custom_hyperparameters": {"max_depth": 4}, + } + + resolved = resolve_model_hyperparameters(config, "classification", "RandomForestClassifier") + + assert resolved["n_estimators"] == 111 + assert resolved["max_depth"] == 4 + + +def test_legacy_nested_hyperparameters_remain_compatible(): + config = { + "config": { + "hyperparameters": {"n_estimators": 77}, + } + } + + resolved = resolve_model_hyperparameters(config, "classification", "RandomForestClassifier") + assert resolved["n_estimators"] == 77 + + +def test_ui_train_ratio_is_converted_to_trainer_test_size(): + resolved = training_config_from_snapshot({"trainTestSplit": 0.85, "randomSeed": 123}) + + assert resolved["test_size"] == 0.15 + assert resolved["random_state"] == 123 + assert resolved["cv_folds"] == 3 diff --git a/Backend/tests/test_sessions.py b/Backend/tests/test_sessions.py new file mode 100644 index 0000000..faa908f --- /dev/null +++ b/Backend/tests/test_sessions.py @@ -0,0 +1,117 @@ +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient + +from app.main import app +from app.services.session_manager import SessionError, SessionManager + + +SESSION_HEADER = "X-NoCodeML-Session" + + +def test_guest_session_api_round_trip(): + with TestClient(app) as client: + created = client.post("/api/v1/session") + assert created.status_code == 201, created.text + payload = created.json() + token = payload["session_token"] + assert payload["temporary"] is True + assert len(token) >= 32 + + missing = client.get("/api/v1/session") + assert missing.status_code == 428 + assert missing.json()["detail"]["code"] == "SESSION_REQUIRED" + + active = client.get("/api/v1/session", headers={SESSION_HEADER: token}) + assert active.status_code == 200, active.text + assert active.json()["status"] == "active" + + cleared = client.delete("/api/v1/session", headers={SESSION_HEADER: token}) + assert cleared.status_code == 204 + + expired = client.get("/api/v1/session", headers={SESSION_HEADER: token}) + assert expired.status_code == 410 + assert expired.json()["detail"]["code"] == "SESSION_EXPIRED" + + +def test_session_workspace_is_hashed_and_isolated(tmp_path: Path): + manager = SessionManager(root=tmp_path, ttl_seconds=3600, close_grace_seconds=30) + token_a, _ = manager.create() + token_b, _ = manager.create() + + workspace_a, _ = manager.resolve(token_a, touch=False) + workspace_b, _ = manager.resolve(token_b, touch=False) + + assert workspace_a != workspace_b + assert workspace_a.name == manager.token_digest(token_a) + assert workspace_b.name == manager.token_digest(token_b) + assert token_a not in str(workspace_a) + assert token_b not in str(workspace_b) + assert set(manager.WORKSPACE_DIRS).issubset({path.name for path in workspace_a.iterdir() if path.is_dir()}) + + dataset_a = manager.safe_path(token_a, "datasets", "sample.csv") + dataset_b = manager.safe_path(token_b, "datasets", "sample.csv") + assert dataset_a != dataset_b + + +def test_session_path_traversal_is_rejected(tmp_path: Path): + manager = SessionManager(root=tmp_path, ttl_seconds=3600, close_grace_seconds=30) + token, _ = manager.create() + + with pytest.raises(SessionError): + manager.safe_path(token, "..", "outside.txt") + + +def test_close_signal_has_grace_and_heartbeat_rescues_session(tmp_path: Path): + manager = SessionManager(root=tmp_path, ttl_seconds=3600, close_grace_seconds=30) + token, _ = manager.create() + + manager.mark_closing(token) + _, closing = manager.resolve(token, touch=False) + assert closing["delete_after"] is not None + + rescued = manager.touch(token) + assert rescued["delete_after"] is None + assert manager.resolve(token, touch=False)[0].exists() + + +def test_cleanup_removes_expired_workspace(tmp_path: Path): + manager = SessionManager(root=tmp_path, ttl_seconds=3600, close_grace_seconds=30) + token, _ = manager.create() + workspace, metadata = manager.resolve(token, touch=False) + metadata["expires_at"] = 0 + manager._write_metadata(workspace, metadata) + + assert manager.cleanup_expired() == 1 + assert not workspace.exists() + + +def test_active_job_lease_blocks_cleanup_until_released(tmp_path: Path): + manager = SessionManager(root=tmp_path, ttl_seconds=3600, close_grace_seconds=30) + token, _ = manager.create() + workspace, metadata = manager.resolve(token, touch=False) + + manager.acquire_job(token) + metadata = manager._read_metadata(workspace) + metadata["expires_at"] = 0 + metadata["delete_after"] = 0 + manager._write_metadata(workspace, metadata) + + assert manager.cleanup_expired() == 0 + assert workspace.exists() + + manager.release_job(token) + assert manager.cleanup_expired() == 1 + assert not workspace.exists() + + +def test_restart_recovery_clears_stale_job_leases(tmp_path: Path): + manager = SessionManager(root=tmp_path, ttl_seconds=3600, close_grace_seconds=30) + token, _ = manager.create() + workspace, _ = manager.resolve(token, touch=False) + manager.acquire_job(token) + + assert manager._read_metadata(workspace)["active_jobs"] == 1 + assert manager.reset_stale_job_leases() == 1 + assert manager._read_metadata(workspace)["active_jobs"] == 0 diff --git a/Backend/tests/test_smoke.py b/Backend/tests/test_smoke.py new file mode 100644 index 0000000..a96280f --- /dev/null +++ b/Backend/tests/test_smoke.py @@ -0,0 +1,75 @@ +from fastapi.testclient import TestClient + +from app.main import app + + +SESSION_HEADER = "X-NoCodeML-Session" + + +def create_session(client: TestClient) -> str: + response = client.post("/api/v1/session") + assert response.status_code == 201, response.text + return response.json()["session_token"] + + +def test_health_endpoint_reports_guest_v3_mode(): + with TestClient(app) as client: + response = client.get("/health") + assert response.status_code == 200 + payload = response.json() + assert payload["status"] == "healthy" + assert payload["version"].startswith("3.") + assert payload["mode"] == "temporary-guest" + + +def test_readiness_is_database_free_and_checks_workspace_and_training(): + with TestClient(app) as client: + response = client.get("/ready") + assert response.status_code == 200, response.text + payload = response.json() + assert payload["status"] == "ready" + assert payload["mode"] == "temporary-guest" + assert payload["checks"]["temporary_workspace"]["status"] == "ready" + assert payload["checks"]["training"]["status"] == "ready" + assert payload["checks"]["training"]["backend"] == "bounded-in-process" + assert "database" not in payload["checks"] + assert "queue" not in payload["checks"] + assert "DATABASE_URL" not in response.text + assert "SECRET_KEY" not in response.text + + +def test_model_catalog_is_available(): + with TestClient(app) as client: + response = client.get("/api/v1/models") + assert response.status_code == 200, response.text + payload = response.json() + assert len(payload["classification"]) == 4 + assert len(payload["regression"]) == 4 + + +def test_persistent_account_and_project_routes_are_not_public(): + with TestClient(app) as client: + assert client.post("/api/v1/auth/register", json={"email": "nobody@example.com", "password": "irrelevant"}).status_code == 404 + assert client.get("/api/v1/datasets").status_code == 404 + assert client.get("/api/v1/experiments").status_code == 404 + assert client.get("/api/v1/training/runs").status_code == 404 + assert client.get("/api/v1/predictions").status_code == 404 + + +def test_ai_assistant_requires_temporary_session_and_fails_safely_without_key(): + payload = { + "system_prompt": "Help interpret derived ML metrics. Never request raw dataset rows.", + "messages": [{"role": "user", "content": "What should I do next?"}], + } + with TestClient(app) as client: + anonymous = client.post("/api/v1/assistant/chat", json=payload) + assert anonymous.status_code == 428 + + token = create_session(client) + configured_guest = client.post( + "/api/v1/assistant/chat", + headers={SESSION_HEADER: token}, + json=payload, + ) + assert configured_guest.status_code == 503 + assert "not configured" in configured_guest.json()["detail"].lower() diff --git a/Backend/tests/test_workspace_datasets.py b/Backend/tests/test_workspace_datasets.py new file mode 100644 index 0000000..d9137a7 --- /dev/null +++ b/Backend/tests/test_workspace_datasets.py @@ -0,0 +1,152 @@ +from io import BytesIO + +from fastapi.testclient import TestClient + +from app.main import app + + +SESSION_HEADER = "X-NoCodeML-Session" + + +def new_session(client: TestClient) -> str: + response = client.post("/api/v1/session") + assert response.status_code == 201, response.text + return response.json()["session_token"] + + +def upload_csv(client: TestClient, token: str, filename: str = "sample.csv"): + csv_bytes = b"age,income,city,target\n21,50000,Bengaluru,1\n25,65000,Hyderabad,0\n29,71000,Chennai,1\n32,83000,Bengaluru,0\n" + return client.post( + "/api/v1/workspace/datasets", + headers={SESSION_HEADER: token}, + files={"file": (filename, BytesIO(csv_bytes), "text/csv")}, + data={"name": "Sample Dataset"}, + ) + + +def test_temporary_dataset_upload_list_preview_delete(): + with TestClient(app) as client: + token = new_session(client) + + uploaded = upload_csv(client, token) + assert uploaded.status_code == 201, uploaded.text + dataset = uploaded.json()["dataset"] + assert dataset["temporary"] is True + assert dataset["row_count"] == 4 + assert dataset["column_count"] == 4 + dataset_id = dataset["id"] + + listed = client.get("/api/v1/workspace/datasets", headers={SESSION_HEADER: token}) + assert listed.status_code == 200, listed.text + assert listed.json()["total"] == 1 + assert listed.json()["datasets"][0]["id"] == dataset_id + + preview = client.get( + f"/api/v1/workspace/datasets/{dataset_id}/preview?rows=2", + headers={SESSION_HEADER: token}, + ) + assert preview.status_code == 200, preview.text + assert preview.json()["columns"] == ["age", "income", "city", "target"] + assert preview.json()["preview_rows"] == 2 + + renamed = client.put( + f"/api/v1/workspace/datasets/{dataset_id}", + headers={SESSION_HEADER: token}, + json={"name": "Renamed Dataset", "description": None}, + ) + assert renamed.status_code == 200, renamed.text + assert renamed.json()["dataset"]["name"] == "Renamed Dataset" + + deleted = client.delete( + f"/api/v1/workspace/datasets/{dataset_id}", + headers={SESSION_HEADER: token}, + ) + assert deleted.status_code == 204 + + listed_after = client.get("/api/v1/workspace/datasets", headers={SESSION_HEADER: token}) + assert listed_after.status_code == 200 + assert listed_after.json()["total"] == 0 + + +def test_temporary_eda_and_plot_generation(): + with TestClient(app) as client: + token = new_session(client) + uploaded = upload_csv(client, token) + assert uploaded.status_code == 201, uploaded.text + dataset_id = uploaded.json()["dataset"]["id"] + + eda = client.get( + f"/api/v1/workspace/datasets/{dataset_id}/eda", + headers={SESSION_HEADER: token}, + ) + assert eda.status_code == 200, eda.text + summary = eda.json() + assert summary["dataset_info"]["temporary"] is True + assert summary["dataset_info"]["row_count"] == 4 + assert "age" in summary["numeric_columns"] + assert "city" in summary["categorical_columns"] + assert summary["missing_data_summary"]["total_missing"] == 0 + + plot = client.post( + f"/api/v1/workspace/datasets/{dataset_id}/plot", + headers={SESSION_HEADER: token}, + json={ + "plot_type": "scatter", + "x_column": "age", + "y_column": "income", + "group_by": "city", + }, + ) + assert plot.status_code == 200, plot.text + payload = plot.json() + assert payload["plot_type"] == "scatter" + assert payload["total_rows"] == 4 + assert payload["data"] + + +def test_temporary_dataset_isolation_between_sessions(): + with TestClient(app) as client: + token_a = new_session(client) + token_b = new_session(client) + + uploaded = upload_csv(client, token_a, "private.csv") + assert uploaded.status_code == 201, uploaded.text + dataset_id = uploaded.json()["dataset"]["id"] + + own = client.get(f"/api/v1/workspace/datasets/{dataset_id}", headers={SESSION_HEADER: token_a}) + assert own.status_code == 200 + + other = client.get(f"/api/v1/workspace/datasets/{dataset_id}", headers={SESSION_HEADER: token_b}) + assert other.status_code == 404 + assert other.json()["detail"]["code"] == "DATASET_NOT_FOUND" + + other_eda = client.get( + f"/api/v1/workspace/datasets/{dataset_id}/eda", + headers={SESSION_HEADER: token_b}, + ) + assert other_eda.status_code == 404 + + other_list = client.get("/api/v1/workspace/datasets", headers={SESSION_HEADER: token_b}) + assert other_list.status_code == 200 + assert other_list.json()["total"] == 0 + + +def test_workspace_rejects_unsupported_and_empty_files(): + with TestClient(app) as client: + token = new_session(client) + + unsupported = client.post( + "/api/v1/workspace/datasets", + headers={SESSION_HEADER: token}, + files={"file": ("notes.txt", BytesIO(b"hello"), "text/plain")}, + ) + assert unsupported.status_code == 400 + assert unsupported.json()["detail"]["code"] == "UNSUPPORTED_FILE_TYPE" + + empty = client.post( + "/api/v1/workspace/datasets", + headers={SESSION_HEADER: token}, + files={"file": ("empty.csv", BytesIO(b"a,b\n"), "text/csv")}, + ) + assert empty.status_code == 400 + assert empty.json()["detail"]["code"] == "EMPTY_DATASET" diff --git a/Backend/tests/test_workspace_training.py b/Backend/tests/test_workspace_training.py new file mode 100644 index 0000000..60c7d30 --- /dev/null +++ b/Backend/tests/test_workspace_training.py @@ -0,0 +1,147 @@ +import time +from io import BytesIO + +from fastapi.testclient import TestClient + +from app.main import app + + +SESSION_HEADER = "X-NoCodeML-Session" + + +def create_session(client: TestClient) -> str: + response = client.post("/api/v1/session") + assert response.status_code == 201, response.text + return response.json()["session_token"] + + +def upload(client: TestClient, token: str, filename: str, content: bytes) -> str: + response = client.post( + "/api/v1/workspace/datasets", + headers={SESSION_HEADER: token}, + files={"file": (filename, BytesIO(content), "text/csv")}, + ) + assert response.status_code == 201, response.text + return response.json()["dataset"]["id"] + + +def wait_for_run(client: TestClient, token: str, run_id: str, timeout: float = 20.0): + deadline = time.time() + timeout + while time.time() < deadline: + response = client.get( + f"/api/v1/workspace/training/runs/{run_id}", + headers={SESSION_HEADER: token}, + ) + assert response.status_code == 200, response.text + run = response.json() + if run["status"] in {"completed", "failed"}: + return run + time.sleep(0.1) + raise AssertionError("Temporary training run did not finish before timeout") + + +def test_guest_classification_training_and_prediction_flow(): + rows = ["age,income,city,churn"] + cities = ["Bengaluru", "Hyderabad", "Chennai"] + for index in range(30): + rows.append(f"{20 + index},{35000 + index * 2500},{cities[index % 3]},{index % 2}") + csv_data = ("\n".join(rows) + "\n").encode() + + with TestClient(app) as client: + token = create_session(client) + dataset_id = upload(client, token, "Customer Churn.csv", csv_data) + started = client.post( + "/api/v1/workspace/training/runs", + headers={SESSION_HEADER: token}, + json={ + "dataset_id": dataset_id, + "target_column": "churn", + "task_type": "classification", + "selected_features": ["age", "income", "city"], + "models": [{"model_type": "LogisticRegression"}], + "test_size": 0.2, + "random_state": 42, + "cv_folds": 3, + }, + ) + assert started.status_code == 202, started.text + run_id = started.json()["id"] + run = wait_for_run(client, token, run_id) + assert run["status"] == "completed", run + assert run["best_model"]["model_type"] == "LogisticRegression" + assert run["best_model"]["model_file"].endswith(".joblib") + assert run["progress"]["percent"] == 100 + assert run["results"][0]["metrics"]["test"]["accuracy"] >= 0 + + single = client.post( + f"/api/v1/workspace/training/runs/{run_id}/predict", + headers={SESSION_HEADER: token}, + json={"features": {"age": 27, "income": 62500, "city": "Bengaluru"}}, + ) + assert single.status_code == 200, single.text + assert str(single.json()["prediction"]) in {"0", "1"} + assert single.json()["model_type"] == "LogisticRegression" + + batch_csv = b"age,income,city\n23,43000,Bengaluru\n31,78000,Hyderabad\n" + batch = client.post( + f"/api/v1/workspace/training/runs/{run_id}/predict/batch", + headers={SESSION_HEADER: token}, + files={"file": ("new-customers.csv", BytesIO(batch_csv), "text/csv")}, + ) + assert batch.status_code == 201, batch.text + batch_payload = batch.json() + assert batch_payload["total_predictions"] == 2 + assert batch_payload["download_filename"].startswith("nocodeml_customer-churn_predictions_") + prediction_id = batch_payload["id"] + + download = client.get( + f"/api/v1/workspace/predictions/{prediction_id}/download", + headers={SESSION_HEADER: token}, + ) + assert download.status_code == 200, download.text + assert "nocodeml_customer-churn_predictions_" in download.headers["content-disposition"] + assert "prediction" in download.text.splitlines()[0] + + +def test_guest_regression_training_flow(): + rows = ["area,bedrooms,city,price"] + cities = ["Bengaluru", "Mysuru"] + for index in range(30): + area = 500 + index * 35 + bedrooms = 1 + (index % 4) + price = 1500000 + area * 4000 + bedrooms * 200000 + rows.append(f"{area},{bedrooms},{cities[index % 2]},{price}") + csv_data = ("\n".join(rows) + "\n").encode() + + with TestClient(app) as client: + token = create_session(client) + dataset_id = upload(client, token, "regression.csv", csv_data) + started = client.post( + "/api/v1/workspace/training/runs", + headers={SESSION_HEADER: token}, + json={ + "dataset_id": dataset_id, + "target_column": "price", + "task_type": "regression", + "selected_features": ["area", "bedrooms", "city"], + "models": [{"model_type": "LinearRegression"}], + "test_size": 0.2, + "random_state": 42, + "cv_folds": 3, + }, + ) + assert started.status_code == 202, started.text + run_id = started.json()["id"] + run = wait_for_run(client, token, run_id) + assert run["status"] == "completed", run + assert run["best_model"]["model_type"] == "LinearRegression" + assert run["best_model"]["metric"] == "r2_score" + assert run["results"][0]["metrics"]["test"]["r2_score"] > 0.5 + + predicted = client.post( + f"/api/v1/workspace/training/runs/{run_id}/predict", + headers={SESSION_HEADER: token}, + json={"features": {"area": 1200, "bedrooms": 3, "city": "Bengaluru"}}, + ) + assert predicted.status_code == 200, predicted.text + assert float(predicted.json()["prediction"]) > 0 diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md new file mode 100644 index 0000000..a0f1a29 --- /dev/null +++ b/DEPLOYMENT.md @@ -0,0 +1,174 @@ +# NoCodeML V3 Deployment + +This release is designed to run without PostgreSQL, Supabase, Redis, Celery or persistent visitor storage. + +Recommended college-project deployment: + +```text +Vercel Hobby (React frontend) + โ†“ HTTPS +Oracle Cloud Always Free Ampere A1 (FastAPI) + โ†“ +RAM-backed temporary session workspaces +``` + +The backend is ARM64-tested in GitHub Actions before release. + +## 1. Backend โ€” Oracle Cloud Always Free + +Use an **Ubuntu 24.04 ARM64** VM with the Always Free `VM.Standard.A1.Flex` shape. Keep the total A1 allocation within the Always Free entitlement shown in your Oracle account. The app is intentionally configured for one bounded training worker. + +During Oracle account creation you may be asked for identity/payment verification. Do not upgrade the account or create paid resources just for NoCodeML. + +### VM networking + +Give the instance a public IPv4 address and allow inbound TCP: + +```text +22 SSH +80 HTTP (Caddy certificate challenge / redirect) +443 HTTPS +``` + +Do **not** expose port 8000 publicly. Caddy is the only public entry point. + +### Clone and start + +```bash +git clone https://github.com/Rishikeshsanin/NoCodeML.git +cd NoCodeML +git switch release/v3-revival +sudo bash deploy/oci/bootstrap-ubuntu.sh +``` + +The bootstrap script: + +- installs Docker and Docker Compose; +- detects the VM's public IPv4 address; +- creates a free `sslip.io` hostname such as `nocodeml-api.203.0.113.10.sslip.io`; +- builds the ARM-compatible backend image; +- starts FastAPI and Caddy; +- obtains HTTPS automatically; +- stores visitor workspaces in tmpfs, not a persistent volume; +- prints the final backend URL. + +Check: + +```text +https:///health +https:///ready +``` + +### Optional Gemini assistant + +Edit `deploy/oci/.env` on the VM and set: + +```env +GEMINI_API_KEY=your_server_side_key +``` + +Never commit that file. The template is tracked; the real `.env` is ignored. + +The core ML workflow works without Gemini. + +## 2. Frontend โ€” Vercel Hobby + +The repository root contains `vercel.json`, so the GitHub repository can be imported directly without changing the project Root Directory. + +Create a **new** Vercel project for `Rishikeshsanin/NoCodeML`. Do not reuse or overwrite another project. + +Set this environment variable for Production, Preview and Development as appropriate: + +```env +VITE_API_URL=https:// +``` + +Then deploy `release/v3-revival` for the release preview. After validation/merge, production should track `main`. + +The build configuration is already versioned: + +```text +Install: npm --prefix Frontend ci +Build: npm --prefix Frontend run build +Output: Frontend/dist +``` + +## 3. Tighten backend CORS + +The bootstrap starts with `FRONTEND_ORIGIN=*` only so the backend can be smoke-tested before the Vercel URL exists. + +As soon as Vercel gives the production URL, edit on the VM: + +```env +FRONTEND_ORIGIN=https://your-nocodeml-project.vercel.app +``` + +Then reload: + +```bash +docker compose --env-file deploy/oci/.env -f deploy/oci/docker-compose.yml up -d +``` + +Verify a browser session can still upload and train. Keeping the exact frontend origin reduces unwanted cross-origin use of the public ML API. + +## 4. Production validation + +Before merging the release branch, complete both flows from the public Vercel URL. + +### Classification + +```text +open site +โ†’ upload CSV +โ†’ inspect EDA +โ†’ choose classification target +โ†’ train at least two models +โ†’ compare results +โ†’ single prediction +โ†’ batch prediction +โ†’ download outputs +โ†’ download session ZIP +โ†’ Clear & restart +``` + +### Regression + +Repeat with a numeric continuous target and verify Rยฒ/MAE/RMSE results plus prediction output. + +Also verify: + +- `/login` and `/register` redirect to the guest workspace; +- no persistent `/api/v1/auth`, `/datasets`, `/experiments`, `/training` or `/predictions` API is exposed; +- two temporary sessions cannot access each other's artifacts; +- clearing a session removes its workspace; +- mobile and desktop layouts do not overflow; +- AI assistant fails gracefully if no Gemini key is configured. + +## 5. Update deployment + +On the VM: + +```bash +cd NoCodeML +git fetch origin +git switch main +git pull --ff-only +docker compose --env-file deploy/oci/.env -f deploy/oci/docker-compose.yml up -d --build +``` + +Vercel can continue auto-deploying from GitHub after `main` becomes the production branch. + +## 6. Rollback + +Recovery branches are intentionally preserved: + +```text +legacy/v2-2026-08-22 +checkpoint/v3-rc-persistent-2026-08-22 +``` + +The public V3 release should normally roll back to the previous V3 release commit/deployment rather than restoring persistent visitor storage. + +## Data-lifecycle note + +Browser close events are not guaranteed to reach a server. NoCodeML therefore combines a close signal with inactivity TTL cleanup. A server/VM/container restart also immediately removes tmpfs visitor workspaces. Caddy's certificate/config volume is persistent, but it contains no user ML data. diff --git a/Frontend/.env.example b/Frontend/.env.example index fbe8d66..83aceb7 100644 --- a/Frontend/.env.example +++ b/Frontend/.env.example @@ -1,17 +1,8 @@ -# =========================================== -# NoCodeML Frontend - Environment Configuration -# =========================================== -# Copy this file to .env and update the values +# NoCodeML Frontend Environment -# Backend API URL -# MUST include the full protocol (http:// or https://) -# Examples: -# Local dev: VITE_API_URL=http://localhost:8000 -# Production: VITE_API_URL=https://api.your-backend-domain.com -# -# WARNING: Omitting the protocol (e.g. VITE_API_URL=nocodeml.cloud) will silently -# route all API calls to the current page's origin instead of the backend. +# Public backend API URL. This value is bundled into the browser build. +# Local: VITE_API_URL=http://localhost:8000 +# Production: VITE_API_URL=https://your-backend.example.com VITE_API_URL=http://localhost:8000 -# Optional: Gemini AI API key for the Data Science Assistant -VITE_GEMINI_API_KEY= +# Do not place secrets or private API keys in VITE_* variables. diff --git a/Frontend/bun.lockb b/Frontend/bun.lockb deleted file mode 100644 index d3914e8..0000000 Binary files a/Frontend/bun.lockb and /dev/null differ diff --git a/Frontend/eslint.config.js b/Frontend/eslint.config.js index 40f72cc..b281407 100644 --- a/Frontend/eslint.config.js +++ b/Frontend/eslint.config.js @@ -21,6 +21,13 @@ export default tseslint.config( ...reactHooks.configs.recommended.rules, "react-refresh/only-export-components": ["warn", { allowConstantExport: true }], "@typescript-eslint/no-unused-vars": "off", + // V2 contains broad API payloads typed as `any`. Keep them visible during the + // V3 migration without letting stylistic legacy debt mask build/runtime regressions. + "@typescript-eslint/no-explicit-any": "warn", + // Some generated shadcn primitives use empty extension interfaces. + "@typescript-eslint/no-empty-object-type": "warn", + // V2 has two harmless mutable declarations that are being cleaned during the type pass. + "prefer-const": "warn", }, }, ); diff --git a/Frontend/src/App.tsx b/Frontend/src/App.tsx index 98ab8c2..d12c317 100644 --- a/Frontend/src/App.tsx +++ b/Frontend/src/App.tsx @@ -1,60 +1,69 @@ -import { Toaster } from "@/components/ui/toaster"; +import { lazy, Suspense } from "react"; +import { Loader2 } from "lucide-react"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { BrowserRouter, Navigate, Route, Routes } from "react-router-dom"; + +import DataScienceAssistant from "@/components/experiments/DataScienceAssistant"; import { Toaster as Sonner } from "@/components/ui/sonner"; +import { Toaster } from "@/components/ui/toaster"; import { TooltipProvider } from "@/components/ui/tooltip"; -import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; -import { BrowserRouter, Routes, Route, Navigate } from "react-router-dom"; -import { AuthProvider } from "./contexts/AuthContext"; -import { ExperimentProvider } from "./contexts/ExperimentContext"; -import { ModelsProvider } from "./contexts/ModelsContext"; -import { TrainingProvider } from "./contexts/TrainingContext"; -import ProtectedRoute from "./components/ProtectedRoute"; import Header from "./components/Header"; -import Home from "./pages/Home"; -import Datasets from "./pages/Datasets"; -import Experiments from "./pages/Experiments"; -import Playground from "./pages/Playground"; -import Login from "./pages/Login"; -import Register from "./pages/Register"; -import NotFound from "./pages/NotFound"; +import { SessionProvider } from "./contexts/SessionContext"; -const queryClient = new QueryClient(); +const Home = lazy(() => import("./pages/Home")); +const Workspace = lazy(() => import("./pages/Workspace")); +const NotFound = lazy(() => import("./pages/NotFound")); + +const queryClient = new QueryClient({ + defaultOptions: { + queries: { + staleTime: 30_000, + retry: 1, + refetchOnWindowFocus: false, + }, + }, +}); + +const PageFallback = () => ( +
+
+ + Loading temporary workspaceโ€ฆ +
+
+); + +const WorkspaceRoute = () => ( + <> + + + +); const App = () => ( - - - - - - - - - } /> - } /> - -
-
- - } /> - } /> - } /> - } /> - } /> - -
- - } - /> - - - - - - + + + + +
+
+ }> + + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + + +
+ + ); diff --git a/Frontend/src/components/Header.tsx b/Frontend/src/components/Header.tsx index 5111933..daf353f 100644 --- a/Frontend/src/components/Header.tsx +++ b/Frontend/src/components/Header.tsx @@ -1,6 +1,7 @@ -import { Link, useLocation } from "react-router-dom"; -import { Activity, LogOut, User } from "lucide-react"; -import { useAuth } from "@/contexts/AuthContext"; +import { Link, useLocation, useNavigate } from "react-router-dom"; +import { Activity, Download, Menu, RefreshCw, ShieldCheck } from "lucide-react"; +import { toast } from "sonner"; + import { Button } from "@/components/ui/button"; import { DropdownMenu, @@ -10,73 +11,80 @@ import { DropdownMenuSeparator, DropdownMenuTrigger, } from "@/components/ui/dropdown-menu"; +import { useSession } from "@/contexts/SessionContext"; +import { workspaceExportAPI } from "@/services/workspaceService"; const Header = () => { const location = useLocation(); - const { user, logout } = useAuth(); - - const navItems = [ - { name: "Home", path: "/" }, - { name: "Datasets", path: "/datasets" }, - { name: "Experiments", path: "/experiments" } - ]; - - const isActive = (path: string) => location.pathname === path; - + const navigate = useNavigate(); + const { status, restartSession } = useSession(); + + const clearSession = async () => { + if (!window.confirm("Clear this temporary session? Download anything you need first.")) return; + try { + await restartSession(); + toast.success("Fresh temporary workspace ready"); + navigate("/workspace"); + } catch (error) { + toast.error(error instanceof Error ? error.message : "Couldn't restart the workspace"); + } + }; + + const downloadSession = async () => { + try { + await workspaceExportAPI.downloadSession(); + } catch (error) { + toast.error(error instanceof Error ? error.message : "Nothing is ready to download yet"); + } + }; + return ( -
-
-
- -
- -
- NoCodeML - - - - -
-
-
- API Connected -
- - - - - - - -
-

My Account

-

{user?.email}

-
- - - - - Log out - - - +
+
+ +
+ +
+
+
NoCodeML
+
Temporary AutoML Studio
+ + + + +
+
+ + {status === "active" ? "Temporary session active" : "Preparing session"} +
+ + + + + + + + + + Temporary workspace + + Home + Workspace + + void downloadSession()}> Download session + void clearSession()}> Clear & restart + +
diff --git a/Frontend/src/components/datasets/DatasetPreviewModal.tsx b/Frontend/src/components/datasets/DatasetPreviewModal.tsx index de02309..2ab2da0 100644 --- a/Frontend/src/components/datasets/DatasetPreviewModal.tsx +++ b/Frontend/src/components/datasets/DatasetPreviewModal.tsx @@ -1,8 +1,10 @@ -import { useState, useEffect } from "react"; +import { useEffect, useState } from "react"; +import { Database, Eye, Rows3 } from "lucide-react"; +import { toast } from "sonner"; + import { Dialog, DialogContent, DialogDescription, DialogHeader, DialogTitle } from "@/components/ui/dialog"; import { Skeleton } from "@/components/ui/skeleton"; -import { datasetAPI } from "@/services/apiService"; -import { toast } from "sonner"; +import { workspaceDatasetAPI, type WorkspaceDatasetPreview } from "@/services/workspaceService"; interface DatasetPreviewModalProps { datasetId: string | null; @@ -13,119 +15,103 @@ interface DatasetPreviewModalProps { const DatasetPreviewModal = ({ datasetId, datasetName, open, onOpenChange }: DatasetPreviewModalProps) => { const [loading, setLoading] = useState(false); - const [previewData, setPreviewData] = useState(null); - const [stats, setStats] = useState(null); + const [previewData, setPreviewData] = useState(null); useEffect(() => { - if (open && datasetId) { - loadData(); + if (!open) { + setPreviewData(null); + return; } - }, [open, datasetId]); - - const loadData = async () => { if (!datasetId) return; - + + let cancelled = false; setLoading(true); - try { - // Backend preview endpoint returns DatasetPreviewResponse: - // { columns: [...], data: [[...], [...]], row_count: number, preview_rows: number } - const preview = await datasetAPI.preview(datasetId, 50); - setPreviewData(preview); - - // Set stats from preview response - setStats({ - totalRows: preview.row_count, - totalColumns: preview.columns?.length || 0, - missingValues: 0, // Not provided in preview response - numericColumns: 0, // Not provided in preview response - columnTypes: {} // Not provided in preview response + workspaceDatasetAPI + .preview(datasetId, 50) + .then((preview) => { + if (!cancelled) setPreviewData(preview); + }) + .catch((error: any) => { + if (!cancelled) toast.error(error.message || "Failed to load dataset preview"); + }) + .finally(() => { + if (!cancelled) setLoading(false); }); - } catch (error: any) { - toast.error(error.message || "Failed to load dataset preview"); - } finally { - setLoading(false); - } - }; + + return () => { + cancelled = true; + }; + }, [open, datasetId]); return ( - + - {datasetName} + {datasetName} - Dataset preview and statistics + Temporary dataset preview. Only the first 50 rows are loaded into this view. {loading && !previewData ? (
- +
+ {[1, 2, 3].map((item) => )} +
- ) : ( -
- {/* Statistics */} - {stats && ( -
-
-

Total Rows

-

{stats.totalRows?.toLocaleString()}

-
-
-

Total Columns

-

{stats.totalColumns}

+ ) : previewData ? ( +
+
+ {[ + { icon: Rows3, label: "Total rows", value: previewData.row_count.toLocaleString() }, + { icon: Database, label: "Columns", value: previewData.columns.length.toLocaleString() }, + { icon: Eye, label: "Preview rows", value: previewData.preview_rows.toLocaleString() }, + ].map(({ icon: Icon, label, value }) => ( +
+
+ {label} +
+

{value}

-
-

Missing Values

-

{stats.missingValues || 0}

-
-
-

Numeric Columns

-

{stats.numericColumns || 0}

-
-
- )} + ))} +
- {/* Data Table */} - {previewData?.data && ( -
-

Data Preview

-
- - - - {previewData.columns?.map((col: string, idx: number) => ( - +
+
+

Data preview

+

Showing {previewData.preview_rows} of {previewData.row_count.toLocaleString()}

+
+
+
- {col} -
+ + + {previewData.columns.map((column) => ( + + ))} + + + + {previewData.data.map((row, rowIndex) => ( + + {row.map((cell, cellIndex) => ( + ))} - - - {previewData.data.map((row: any[], rowIdx: number) => ( - - {row.map((cell: any, cellIdx: number) => ( - - ))} - - ))} - -
+ {column} +
+ {cell !== null && cell !== undefined && cell !== "" ? String(cell) : ( + null + )} +
- {cell !== null && cell !== undefined ? String(cell) : ( - null - )} -
-
- - {/* Preview Info */} -
-

- Showing {previewData.preview_rows} of {stats?.totalRows?.toLocaleString() || 0} total rows -

-
+ ))} + +
- )} +
+ ) : ( +
No preview is available for this dataset.
)}
diff --git a/Frontend/src/components/datasets/DatasetRenameModal.tsx b/Frontend/src/components/datasets/DatasetRenameModal.tsx index afe1644..017a671 100644 --- a/Frontend/src/components/datasets/DatasetRenameModal.tsx +++ b/Frontend/src/components/datasets/DatasetRenameModal.tsx @@ -1,13 +1,14 @@ -import { useState } from "react"; -import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from "@/components/ui/dialog"; +import { useEffect, useState } from "react"; +import { toast } from "sonner"; + import { Button } from "@/components/ui/button"; +import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from "@/components/ui/dialog"; import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; -import { datasetAPI } from "@/services/apiService"; -import { toast } from "sonner"; +import { workspaceDatasetAPI } from "@/services/workspaceService"; interface DatasetRenameModalProps { - dataset: { id: string; name: string } | null; + dataset: { id: string; name: string; description?: string | null } | null; open: boolean; onOpenChange: (open: boolean) => void; onRenameSuccess: () => void; @@ -17,22 +18,24 @@ const DatasetRenameModal = ({ dataset, open, onOpenChange, onRenameSuccess }: Da const [newName, setNewName] = useState(""); const [submitting, setSubmitting] = useState(false); - const handleSubmit = async (e: React.FormEvent) => { - e.preventDefault(); - + useEffect(() => { + if (open) setNewName(dataset?.name || ""); + }, [open, dataset]); + + const handleSubmit = async (event: React.FormEvent) => { + event.preventDefault(); if (!newName.trim() || !dataset) { toast.error("Please enter a valid name"); return; } - + setSubmitting(true); try { - await datasetAPI.update(dataset.id, { + await workspaceDatasetAPI.update(dataset.id, { name: newName.trim(), - description: null // Keep existing description or set to null + description: dataset.description ?? null, }); - toast.success("Dataset renamed successfully"); - setNewName(""); + toast.success("Dataset renamed"); onRenameSuccess(); onOpenChange(false); } catch (error: any) { @@ -47,30 +50,32 @@ const DatasetRenameModal = ({ dataset, open, onOpenChange, onRenameSuccess }: Da
- Rename Dataset + Rename dataset - Enter a new name for "{dataset?.name}" + This only changes the display name inside your temporary session. - +
- + setNewName(e.target.value)} + onChange={(event) => setNewName(event.target.value)} placeholder={dataset?.name} disabled={submitting} + maxLength={200} className="mt-2" + autoFocus />
- + -
diff --git a/Frontend/src/components/datasets/DatasetUploadModal.tsx b/Frontend/src/components/datasets/DatasetUploadModal.tsx index 202a2c5..f6599e4 100644 --- a/Frontend/src/components/datasets/DatasetUploadModal.tsx +++ b/Frontend/src/components/datasets/DatasetUploadModal.tsx @@ -1,79 +1,69 @@ -import { useState, useCallback } from "react"; -import { Dialog, DialogContent, DialogDescription, DialogHeader, DialogTitle } from "@/components/ui/dialog"; -import { Button } from "@/components/ui/button"; -import { Upload, X, FileUp } from "lucide-react"; -import { datasetAPI } from "@/services/apiService"; +import { useCallback, useState } from "react"; +import { FileUp, ShieldCheck, Upload, X } from "lucide-react"; import { toast } from "sonner"; +import { Button } from "@/components/ui/button"; +import { Dialog, DialogContent, DialogDescription, DialogHeader, DialogTitle } from "@/components/ui/dialog"; +import { workspaceDatasetAPI } from "@/services/workspaceService"; + interface DatasetUploadModalProps { open: boolean; onOpenChange: (open: boolean) => void; onUploadSuccess: () => void; } +const ALLOWED_EXTENSIONS = [".csv", ".xlsx", ".xls", ".parquet"]; + const DatasetUploadModal = ({ open, onOpenChange, onUploadSuccess }: DatasetUploadModalProps) => { const [file, setFile] = useState(null); const [isDragging, setIsDragging] = useState(false); const [uploading, setUploading] = useState(false); - const validateFile = (file: File): string | null => { - const maxSize = 100 * 1024 * 1024; // 100MB - - if (!file.name.endsWith('.csv')) { - return "Only CSV files are supported"; - } - - if (file.size > maxSize) { - return "File size must be less than 100MB"; - } - - if (file.size === 0) { - return "File is empty"; + const validateFile = (selectedFile: File): string | null => { + const maxSize = 100 * 1024 * 1024; + const lowerName = selectedFile.name.toLowerCase(); + + if (!ALLOWED_EXTENSIONS.some((extension) => lowerName.endsWith(extension))) { + return "Use a CSV, Excel or Parquet dataset."; } - + if (selectedFile.size > maxSize) return "File size must be 100 MB or less."; + if (selectedFile.size === 0) return "This file is empty."; return null; }; const handleFileSelect = (selectedFile: File) => { const error = validateFile(selectedFile); - if (error) { toast.error(error); return; } - setFile(selectedFile); }; - const handleDrop = useCallback((e: React.DragEvent) => { - e.preventDefault(); + const handleDrop = useCallback((event: React.DragEvent) => { + event.preventDefault(); setIsDragging(false); - - const droppedFile = e.dataTransfer.files[0]; - if (droppedFile) { - handleFileSelect(droppedFile); - } + const droppedFile = event.dataTransfer.files[0]; + if (droppedFile) handleFileSelect(droppedFile); }, []); - const handleDragOver = useCallback((e: React.DragEvent) => { - e.preventDefault(); + const handleDragOver = useCallback((event: React.DragEvent) => { + event.preventDefault(); setIsDragging(true); }, []); - const handleDragLeave = useCallback((e: React.DragEvent) => { - e.preventDefault(); + const handleDragLeave = useCallback((event: React.DragEvent) => { + event.preventDefault(); setIsDragging(false); }, []); const handleUpload = async () => { if (!file) return; - setUploading(true); try { - // Extract name from filename without extension const fileName = file.name.replace(/\.[^/.]+$/, ""); - await datasetAPI.upload(file, fileName); - toast.success("Dataset uploaded successfully"); + await workspaceDatasetAPI.upload(file, fileName); + toast.success("Dataset added to your temporary workspace"); setFile(null); onUploadSuccess(); onOpenChange(false); @@ -85,63 +75,64 @@ const DatasetUploadModal = ({ open, onOpenChange, onUploadSuccess }: DatasetUplo }; const handleClose = () => { - if (!uploading) { - setFile(null); - onOpenChange(false); - } + if (uploading) return; + setFile(null); + onOpenChange(false); }; return ( - + - Upload Dataset + Upload a dataset - Upload a CSV file to create a new dataset (max 100MB) + CSV, Excel and Parquet files are supported up to 100 MB. -
+
+
+ +

Your upload stays in this temporary NoCodeML session and is removed when the session is cleared or expires.

+
+
+ +
{!file ? (
-
- +
+
-
-

- Drop your CSV file here -

-

- or click to browse -

- +

Drop your dataset here

+

or choose a file from your device

- { - const selectedFile = e.target.files?.[0]; + accept=".csv,.xlsx,.xls,.parquet" + onChange={(event) => { + const selectedFile = event.target.files?.[0]; if (selectedFile) handleFileSelect(selectedFile); + event.currentTarget.value = ""; }} className="hidden" /> @@ -149,28 +140,20 @@ const DatasetUploadModal = ({ open, onOpenChange, onUploadSuccess }: DatasetUplo
) : ( -
-
-
-
- +
+
+
+
+
- -
-

{file.name}

-

- {(file.size / (1024 * 1024)).toFixed(2)} MB -

+
+

{file.name}

+

{(file.size / (1024 * 1024)).toFixed(2)} MB

- {!uploading && ( - )}
@@ -178,20 +161,10 @@ const DatasetUploadModal = ({ open, onOpenChange, onUploadSuccess }: DatasetUplo )}
-
- - +
diff --git a/Frontend/src/components/experiments/DataScienceAssistant.tsx b/Frontend/src/components/experiments/DataScienceAssistant.tsx index fe89ecf..2953365 100644 --- a/Frontend/src/components/experiments/DataScienceAssistant.tsx +++ b/Frontend/src/components/experiments/DataScienceAssistant.tsx @@ -1,1072 +1,191 @@ -import { useState, useEffect, useRef } from 'react'; -import { X, MessageCircle, Send, Loader2 } from 'lucide-react'; -import { Button } from '@/components/ui/button'; -import { Input } from '@/components/ui/input'; -import { Card } from '@/components/ui/card'; -import { useEDA } from '@/hooks/useEDA'; -import { useExperiment } from '@/contexts/ExperimentContext'; -import ReactMarkdown from 'react-markdown'; +import { useEffect, useRef, useState } from "react"; +import { Bot, Loader2, MessageCircle, Send, Sparkles, X } from "lucide-react"; +import ReactMarkdown from "react-markdown"; + +import { Button } from "@/components/ui/button"; +import { Card } from "@/components/ui/card"; +import { Input } from "@/components/ui/input"; +import { useSession } from "@/contexts/SessionContext"; +import { API_BASE_URL, getStoredSessionToken } from "@/services/sessionService"; +import { workspaceDatasetAPI, workspaceTrainingAPI } from "@/services/workspaceService"; interface Message { - role: 'user' | 'assistant'; + role: "user" | "assistant"; content: string; } interface DataScienceAssistantProps { datasetId?: string; - edaData?: any; // EDA data can be passed directly or fetched via hook - currentPhase?: 'analysis' | 'config' | 'training' | 'results' | 'predict'; - experimentConfig?: any; // Current experiment configuration - trainingData?: any; // Training runs and metrics - resultsData?: any; // Final results and model comparisons + edaData?: unknown; + currentPhase?: "analysis" | "config" | "training" | "results" | "predict"; + experimentConfig?: unknown; + trainingData?: unknown; + resultsData?: unknown; } -export const DataScienceAssistant = ({ - datasetId, - edaData: propEdaData, - currentPhase = 'analysis', - experimentConfig, - trainingData, - resultsData -}: DataScienceAssistantProps) => { - - // Determine welcome message based on phase - const getWelcomeMessage = () => { - switch (currentPhase) { - case 'analysis': - return '๐Ÿ‘‹ Hello! I\'m your Data Science Assistant. I can help you understand your data quality, distributions, and what insights your dataset reveals. Ask me anything!'; - case 'config': - return '๐ŸŽฏ Hi! I can help you choose the best models, select important features, and configure your experiment. What would you like to know?'; - case 'training': - return '๐Ÿš€ Hey! I\'m here to help you understand training progress, interpret metrics, and troubleshoot any issues. Ask away!'; - case 'results': - return '๐Ÿ“Š Hello! I can help you interpret model performance, compare results, and understand which model works best for your use case.'; - case 'predict': - return '๐Ÿ”ฎ Hi! I can help you understand predictions, confidence scores, and how to use your trained models.'; - default: - return '๐Ÿ‘‹ Hello! I\'m your Data Science Assistant. Ask me anything about your ML experiment!'; - } - }; +const safeJson = (value: unknown, maxLength = 14000) => { + try { + const text = JSON.stringify(value, null, 2); + return text.length <= maxLength ? text : `${text.slice(0, maxLength)}\nโ€ฆcontext truncated`; + } catch { + return "Context unavailable."; + } +}; +export const DataScienceAssistant = (props: DataScienceAssistantProps = {}) => { + void props; + const { status } = useSession(); const [isOpen, setIsOpen] = useState(false); + const [input, setInput] = useState(""); + const [isLoading, setIsLoading] = useState(false); const [messages, setMessages] = useState([ { - role: 'assistant', - content: getWelcomeMessage() - } + role: "assistant", + content: "Hi โ€” I can help interpret your current NoCodeML workspace, model results and next steps. I use derived workspace context, not raw dataset rows.", + }, ]); - const [input, setInput] = useState(''); - const [isLoading, setIsLoading] = useState(false); - const [position, setPosition] = useState({ x: 0, y: 0 }); - const [isDragging, setIsDragging] = useState(false); - const [dragStart, setDragStart] = useState({ x: 0, y: 0 }); - const [chatSize, setChatSize] = useState({ width: 384, height: 500 }); // Default: w-96 (384px), h-[500px] - const [isResizing, setIsResizing] = useState(false); - const [resizeDirection, setResizeDirection] = useState<'corner-br' | 'corner-bl' | 'corner-tr' | 'corner-tl' | 'bottom' | 'top' | 'right' | 'left' | null>(null); - const [resizeStart, setResizeStart] = useState({ x: 0, y: 0, width: 0, height: 0 }); const messagesEndRef = useRef(null); - const buttonRef = useRef(null); - const chatRef = useRef(null); - - const { currentExperiment } = useExperiment(); - const { edaData: hookEdaData } = useEDA(datasetId || currentExperiment?.datasetId); - - // Use provided EDA data or fetched data - const edaData = propEdaData || hookEdaData; - - // Gemini API Configuration - const GEMINI_API_KEY = import.meta.env.VITE_GEMINI_API_KEY || ''; - const GEMINI_API_URL = `https://generativelanguage.googleapis.com/v1beta/models/gemini-2.0-flash-exp:streamGenerateContent?alt=sse&key=${GEMINI_API_KEY}`; - - // Update welcome message when phase changes - useEffect(() => { - setMessages([{ - role: 'assistant', - content: getWelcomeMessage() - }]); - }, [currentPhase]); useEffect(() => { - messagesEndRef.current?.scrollIntoView({ behavior: 'smooth' }); - }, [messages]); - - // Generate phase-specific context for the AI - const generatePhaseContext = () => { - let context = ''; - - // Always include experiment basics if available - if (currentExperiment) { - context += `# Current Experiment: ${currentExperiment.name}\n`; - context += `Dataset: ${currentExperiment.datasetName}\n\n`; - } - - // Phase-specific context - switch (currentPhase) { - case 'analysis': - context += generateEDAContext(); - break; - case 'config': - context += generateConfigContext(); - break; - case 'training': - context += generateTrainingContext(); - break; - case 'results': - context += generateResultsContext(); - break; - case 'predict': - context += generatePredictionContext(); - break; - } - - return context; - }; - - // Generate comprehensive data analysis inferences from EDA data - const generateEDAContext = () => { - if (!edaData) { - return 'No data analysis available yet. Please complete the EDA step first.'; - } - - const { dataset_info, columns, numeric_columns, categorical_columns, statistics, correlations, missing_data_summary } = edaData; - - let inferences = `# Dataset Analysis Summary - -## Dataset Overview -- **Name**: ${dataset_info.name} -- **Total Rows**: ${dataset_info.row_count.toLocaleString()} -- **Total Columns**: ${dataset_info.column_count} -- **Memory Usage**: ${(dataset_info.memory_usage_bytes / (1024 * 1024)).toFixed(2)} MB -- **Missing Data**: ${missing_data_summary.missing_percent.toFixed(2)}% (${missing_data_summary.total_missing.toLocaleString()} out of ${missing_data_summary.total_cells.toLocaleString()} cells) - -## Data Quality Assessment - -### Missing Data Analysis -`; - - if (missing_data_summary.columns_with_missing.length > 0) { - missing_data_summary.columns_with_missing.forEach((col: any) => { - inferences += `- **${col.column}**: ${col.missing_count} missing values (${col.missing_percent.toFixed(2)}%)\n`; - }); - } else { - inferences += '- No missing data detected!\n'; - } - - inferences += ` -### Detected ID Columns -`; - const idColumns = columns.filter((col: any) => col.is_id_column); - if (idColumns.length > 0) { - idColumns.forEach((col: any) => { - inferences += `- **${col.name}**: Unique identifier (${col.unique_count.toLocaleString()} unique values)\n`; - }); - } else { - inferences += '- No ID columns detected\n'; - } - - inferences += ` -## Column Classification - -### Numeric Features (${numeric_columns.length} columns) -`; - numeric_columns.forEach((col: string, idx: number) => { - const colInfo = columns.find((c: any) => c.name === col); - const stats = statistics[col]; - if (stats) { - inferences += `${idx + 1}. **${col}**: mean: ${stats.mean?.toFixed(2)}, std: ${stats.std?.toFixed(2)}, range: [${stats.min?.toFixed(2)}, ${stats.max?.toFixed(2)}]\n`; - } - }); - - inferences += ` -### Categorical Features (${categorical_columns.length} columns) -`; - categorical_columns.forEach((col: string, idx: number) => { - const colInfo = columns.find((c: any) => c.name === col); - if (colInfo) { - inferences += `${idx + 1}. **${col}**: ${colInfo.unique_count} unique values, ${colInfo.missing_count} missing\n`; - } - }); - - if (correlations && correlations.pairs && correlations.pairs.length > 0) { - inferences += ` -## Correlation Analysis - -### Strong Correlations (|r| > 0.7) -`; - correlations.pairs.forEach((pair: any) => { - inferences += `- **${pair.col1} โ†” ${pair.col2}**: r = ${pair.correlation.toFixed(3)}\n`; - }); - } - - inferences += ` -## Model Recommendations - -Based on your data characteristics, here are the models available in our platform: - -### For CLASSIFICATION Tasks: - -#### 1. LightGBM (Light Gradient Boosting Machine) - HIGHLY RECOMMENDED โญโญโญ -**Why this model is best for your data:** -- **Speed**: Extremely fast training, especially with ${dataset_info.row_count.toLocaleString()} rows -- **Memory efficient**: Uses less RAM than other gradient boosting methods -- **Handles your data perfectly**: Works great with ${numeric_columns.length} numeric and ${categorical_columns.length} categorical features -- **Missing data friendly**: Native support for ${missing_data_summary.missing_percent.toFixed(2)}% missing values (no preprocessing needed!) -- **Feature importance**: Tells you which columns matter most for predictions - -**Technical Details:** -- Algorithm: Gradient boosting with leaf-wise tree growth -- Best for: Large datasets (>1000 rows), mixed data types -- Handles: Imbalanced classes, outliers, non-linear relationships - -**Expected Performance:** -- Accuracy: 89-95% -- Training Time: Fast (seconds to minutes) -- ROC-AUC: 0.92-0.97 - -**When to use:** Perfect for production systems needing speed and accuracy - ---- - -#### 2. XGBoost (Extreme Gradient Boosting) - HIGHLY RECOMMENDED โญโญโญ -**Why this model is best for your data:** -- **Most powerful**: Often wins machine learning competitions -- **Robust**: Handles outliers and messy data extremely well -- **Versatile**: Works with any type of data (${numeric_columns.length} numeric, ${categorical_columns.length} categorical) -- **Regularization**: Built-in protection against overfitting -- **Missing values**: Learns best direction for missing data automatically - -**Technical Details:** -- Algorithm: Gradient boosting with depth-wise tree growth -- Best for: When you need maximum accuracy, any dataset size -- Handles: Complex patterns, feature interactions, non-linear relationships - -**Expected Performance:** -- Accuracy: 90-96% -- Training Time: Moderate (slightly slower than LightGBM) -- ROC-AUC: 0.93-0.98 - -**When to use:** When accuracy is top priority and training time is flexible - ---- - -#### 3. Random Forest - RECOMMENDED โญโญ -**Why this model is good for your data:** -- **Easy to understand**: Simple concept (many decision trees voting) -- **No tuning needed**: Works well with default settings -- **Stable**: Less sensitive to parameter choices -- **Feature importance**: Shows which columns are most important -- **No scaling required**: Can use your data as-is - -**Technical Details:** -- Algorithm: Ensemble of decision trees with bootstrap sampling -- Best for: Quick baseline, interpretable models, robust predictions -- Handles: ${dataset_info.row_count.toLocaleString()} rows very well - -**Expected Performance:** -- Accuracy: 85-92% -- Training Time: Moderate -- ROC-AUC: 0.88-0.94 - -**When to use:** Need reliable results quickly without much tuning - ---- - -#### 4. Logistic Regression - BASELINE โญ -**Why this model might work for your data:** -- **Highly interpretable**: Easy to explain to stakeholders -- **Fast**: Trains in seconds even on large data -- **Simple**: Straightforward coefficients show feature impact -- **Good baseline**: Helps evaluate if complex models are needed - -**Limitations for your data:** -- **Linear only**: Cannot capture complex patterns in ${numeric_columns.length} features -- **Needs preprocessing**: Requires scaling and encoding -- **Correlation issues**: May struggle with correlated features - -**Technical Details:** -- Algorithm: Linear model with sigmoid activation -- Best for: Linear relationships, small datasets, highly interpretable needs -- Struggles with: Non-linear patterns, feature interactions - -**Expected Performance:** -- Accuracy: 75-85% -- Training Time: Very fast (seconds) -- ROC-AUC: 0.80-0.88 - -**When to use:** Quick baseline or when model interpretability is critical - ---- - -### For REGRESSION Tasks: - -#### 1. LightGBM (Light Gradient Boosting Machine) - HIGHLY RECOMMENDED โญโญโญ -**Why this model is best for your data:** -- **Fast and accurate**: Best combination of speed and performance -- **Handles ${numeric_columns.length} features** efficiently -- **Missing data support**: ${missing_data_summary.missing_percent.toFixed(2)}% missing values? No problem! -- **Continuous predictions**: Excellent for any range of target values - -**Expected Performance:** -- Rยฒ Score: 0.87-0.94 -- RMSE: Low to Very Low -- Training Time: Fast - ---- - -#### 2. XGBoost (Extreme Gradient Boosting) - HIGHLY RECOMMENDED โญโญโญ -**Why this model is best for your data:** -- **Maximum accuracy**: Industry standard for regression -- **Robust to outliers**: Won't be thrown off by extreme values -- **Feature interactions**: Automatically discovers relationships between ${numeric_columns.length} features -- **Regularization**: Prevents overfitting on ${dataset_info.row_count.toLocaleString()} samples - -**Expected Performance:** -- Rยฒ Score: 0.88-0.95 -- RMSE: Very Low -- Training Time: Moderate - ---- - -#### 3. Random Forest - RECOMMENDED โญโญ -**Why this model is good for your data:** -- **Reliable**: Consistent performance across different datasets -- **No scaling needed**: Use raw feature values -- **Handles ${dataset_info.row_count.toLocaleString()} rows** well -- **Uncertainty estimates**: Can provide prediction confidence intervals - -**Expected Performance:** -- Rยฒ Score: 0.82-0.91 -- RMSE: Low -- Training Time: Moderate - ---- - -#### 4. Linear Regression - BASELINE โญ -**Why this model might work:** -- **Interpretable**: Clear coefficient for each feature -- **Fast**: Trains instantly -- **Simple**: Easy to understand and explain - -**Limitations:** -- **Linear relationships only**: Assumes straight-line relationships -- **Sensitive to outliers**: Extreme values can skew predictions -- **Needs feature engineering**: May require creating interaction terms manually - -**Expected Performance:** -- Rยฒ Score: 0.70-0.82 (if relationships are linear) -- RMSE: Moderate -- Training Time: Very fast - ---- - -## Quick Decision Guide: - -**๐ŸŽฏ For Maximum Accuracy:** Choose **XGBoost** or **LightGBM** -**โšก For Speed + Accuracy:** Choose **LightGBM** -**๐Ÿ” For Interpretability:** Choose **Logistic/Linear Regression** or **Random Forest** -**๐Ÿš€ For Quick Baseline:** Choose **Random Forest** -**๐Ÿ“Š For Production Systems:** Choose **LightGBM** (fast inference) - -## Feature Engineering Recommendations - -### Handle Missing Data: -`; - - if (missing_data_summary.columns_with_missing.length > 0) { - missing_data_summary.columns_with_missing.forEach((col: any) => { - inferences += `- **${col.column}**: Consider imputation strategy (mean/median for numeric, mode for categorical)\n`; - }); - } - - inferences += ` -### Feature Scaling: -- Normalize/standardize numeric features for distance-based algorithms -- Not required for tree-based models (Random Forest, XGBoost) - -### Categorical Encoding: -- Use one-hot encoding for ${categorical_columns.length} categorical features -- Consider target encoding for high-cardinality categories - -## Training Configuration - -### Recommended Split: -- **Train**: 70% (${Math.floor(dataset_info.row_count * 0.7).toLocaleString()} samples) -- **Validation**: 15% (${Math.floor(dataset_info.row_count * 0.15).toLocaleString()} samples) -- **Test**: 15% (${Math.floor(dataset_info.row_count * 0.15).toLocaleString()} samples) - -### Cross-Validation: -- **Method**: 5-Fold Stratified K-Fold (for classification) -- **Metric**: ROC-AUC for classification, Rยฒ for regression -`; - - return inferences; - }; - - // Generate Model Config phase context - const generateConfigContext = () => { - let context = '# Model Configuration Phase\n\n'; - - if (edaData) { - const { dataset_info, numeric_columns, categorical_columns } = edaData; - context += `## Dataset Summary\n`; - context += `- Total Features: ${numeric_columns.length + categorical_columns.length}\n`; - context += `- Numeric: ${numeric_columns.length}, Categorical: ${categorical_columns.length}\n`; - context += `- Total Rows: ${dataset_info.row_count.toLocaleString()}\n\n`; - } - - if (experimentConfig) { - context += `## Current Configuration\n`; - context += `- Task Type: ${experimentConfig.taskType || 'Not set'}\n`; - context += `- Target Column: ${experimentConfig.targetColumn || 'Not selected'}\n`; - context += `- Selected Features: ${experimentConfig.selectedFeatures?.length || 0} features\n`; - if (experimentConfig.selectedFeatures?.length > 0) { - context += ` - Features: ${experimentConfig.selectedFeatures.slice(0, 10).join(', ')}${experimentConfig.selectedFeatures.length > 10 ? '...' : ''}\n`; - } - context += `- Models Selected: ${experimentConfig.models?.length || 0}\n`; - if (experimentConfig.models?.length > 0) { - context += ` - Models: ${experimentConfig.models.map((m: any) => m.name).join(', ')}\n`; - } - context += `\n`; - } - - context += `## Available Models\n`; - context += `**Classification:** Logistic Regression, Random Forest, XGBoost, LightGBM\n`; - context += `**Regression:** Linear Regression, Random Forest, XGBoost, LightGBM\n\n`; - context += `**Recommendation:** For ${edaData?.dataset_info?.row_count > 10000 ? 'large' : 'medium'} datasets, LightGBM or XGBoost typically perform best.\n`; - - return context; - }; - - // Generate Training phase context - const generateTrainingContext = () => { - let context = '# Training Phase\n\n'; - - if (trainingData && trainingData.length > 0) { - context += `## Training Status\n`; - context += `- Total Models Training: ${trainingData.length}\n\n`; - - trainingData.forEach((run: any, idx: number) => { - context += `### Model ${idx + 1}: ${run.modelName || 'Unknown'}\n`; - context += `- Status: ${run.status}\n`; - if (run.metrics) { - context += `- Current Metrics:\n`; - Object.entries(run.metrics).forEach(([key, value]) => { - context += ` - ${key}: ${typeof value === 'number' ? value.toFixed(4) : value}\n`; - }); - } - if (run.progress) { - context += `- Progress: ${run.progress}%\n`; - } - context += `\n`; - }); - } else { - context += '## No Active Training\n'; - context += 'Training has not started yet or no training data available.\n\n'; - } - - context += `## Training Tips\n`; - context += `- **Loss increasing?** May indicate learning rate too high or overfitting\n`; - context += `- **Slow convergence?** Try increasing learning rate or checking feature scaling\n`; - context += `- **Perfect training accuracy?** Watch for overfitting - check validation metrics\n`; - - return context; - }; - - // Generate Results phase context - const generateResultsContext = () => { - let context = '# Results Analysis\n\n'; - - if (resultsData && resultsData.modelResults) { - context += `## Model Performance Summary\n`; - context += `- Total Models Trained: ${resultsData.modelResults.length}\n\n`; - - // Sort by primary metric (accuracy or Rยฒ) - const sortedResults = [...resultsData.modelResults].sort((a, b) => { - const metricA = a.metrics?.accuracy || a.metrics?.r2_score || 0; - const metricB = b.metrics?.accuracy || b.metrics?.r2_score || 0; - return metricB - metricA; - }); - - sortedResults.forEach((result: any, idx: number) => { - context += `### ${idx + 1}. ${result.modelName}\n`; - if (result.metrics) { - Object.entries(result.metrics).forEach(([key, value]) => { - context += `- ${key}: ${typeof value === 'number' ? value.toFixed(4) : value}\n`; - }); - } - context += `\n`; - }); - - if (resultsData.bestModel) { - context += `## ๐Ÿ† Best Model: ${resultsData.bestModel.name}\n`; - context += `This model achieved the highest performance on your validation set.\n\n`; - } - } else { - context += '## No Results Available\n'; - context += 'Training needs to be completed to see results.\n\n'; - } - - context += `## Metrics Explained\n`; - context += `- **Accuracy**: % of correct predictions (higher is better)\n`; - context += `- **Precision**: Of predicted positives, how many are actually positive?\n`; - context += `- **Recall**: Of actual positives, how many did we catch?\n`; - context += `- **F1 Score**: Balance between precision and recall\n`; - context += `- **ROC-AUC**: Model's ability to distinguish between classes (0.5 = random, 1.0 = perfect)\n`; - - return context; - }; - - // Generate Prediction phase context - const generatePredictionContext = () => { - let context = '# Prediction Phase\n\n'; - - if (resultsData?.bestModel) { - context += `## Selected Model: ${resultsData.bestModel.name}\n`; - context += `This model will be used for predictions.\n\n`; - } - - context += `## Making Predictions\n`; - context += `- Upload new data or use sample data\n`; - context += `- Ensure data has same features as training data\n`; - context += `- Model will output predicted values with confidence scores\n\n`; - - context += `## Interpreting Predictions\n`; - context += `- **Confidence Score**: Model's certainty (0-100%)\n`; - context += `- **Low confidence?** Model is uncertain - verify input data quality\n`; - context += `- **Unexpected results?** Check if input data matches training distribution\n`; - - return context; - }; - - const systemPrompt = `You are an expert Data Science Assistant designed to help BOTH technical and non-technical users understand their data and make informed decisions about machine learning models. - -YOUR COMMUNICATION STYLE: -- Start with simple, clear explanations (for non-technical users) -- Then provide technical details (for technical users) -- Use analogies and real-world examples when helpful -- Avoid jargon unless explaining technical concepts -- Be encouraging and supportive - -CRITICAL RULES: -1. Base ALL your answers EXCLUSIVELY on the context provided below. -2. If a question cannot be answered from the provided data, clearly state: "I don't have enough information in the current context to answer that question." -3. Never make up or assume information that isn't in the provided context. -4. Always explain WHY you recommend something, not just WHAT to do. -5. Keep responses concise (max 250 words) unless user asks for detailed explanation. - -CURRENT PHASE: ${currentPhase.toUpperCase()} - -YOUR PRIMARY TASKS: - -**For Non-Technical Users:** -- Explain what the data shows in plain English -- Recommend models using simple language (e.g., "This model is like having 1000 experts vote on the answer") -- Explain expected results in business terms ("You can expect 90% accuracy, meaning 9 out of 10 predictions will be correct") -- Provide actionable next steps - -**For Technical Users:** -- Provide detailed technical specifications -- Explain algorithms and mathematical concepts -- Discuss hyperparameters and optimization strategies -- Compare model architectures and computational complexity - -**Questions You Should Handle:** - -1. **Model Selection Questions:** - - "Which model is best for my data?" โ†’ Recommend based on dataset characteristics with clear reasoning - - "Why should I use XGBoost over Random Forest?" โ†’ Explain differences and when each excels - - "What's the difference between LightGBM and XGBoost?" โ†’ Compare speed, accuracy, memory usage - - "Is Logistic Regression good enough?" โ†’ Evaluate based on data complexity - -2. **Data Quality Questions:** - - "Do I have too much missing data?" โ†’ Assess missing data percentage and provide guidance - - "Should I remove outliers?" โ†’ Advise based on model robustness - - "Is my dataset too small?" โ†’ Evaluate dataset size for model requirements - -3. **Feature Questions:** - - "What features are most important?" โ†’ Reference correlations and feature types - - "Should I create new features?" โ†’ Suggest based on relationships in data - - "Can I use all columns?" โ†’ Identify ID columns to exclude - -4. **Performance Questions:** - - "What accuracy can I expect?" โ†’ Provide ranges based on data characteristics - - "How long will training take?" โ†’ Estimate based on data size and model choice - - "Will my model overfit?" โ†’ Assess based on data size and complexity - -5. **Non-Technical Questions:** - - "What does this data tell me?" โ†’ Summarize key insights in simple terms - - "How do I know if my model is good?" โ†’ Explain metrics in plain language - - "What should I do next?" โ†’ Provide step-by-step guidance - - "Is this hard to do?" โ†’ Reassure and explain the process - -6. **Comparison Questions:** - - "LightGBM vs XGBoost - which one?" โ†’ Compare for their specific data - - "Should I use ensemble models?" โ†’ Explain benefits for their use case - - "Linear vs non-linear models?" โ†’ Assess data relationships - -**AVAILABLE MODELS (Always mention these are the models in our platform):** - -Classification: Logistic Regression, Random Forest, XGBoost, LightGBM -Regression: Linear Regression, Random Forest, XGBoost, LightGBM - -**Model Recommendation Framework:** -1. **First choice (Best accuracy + speed)**: LightGBM or XGBoost -2. **Safe choice (Reliable, easy)**: Random Forest -3. **Baseline (Quick test)**: Logistic/Linear Regression -4. Always explain WHY each model suits their specific data - -CONTEXT FOR CURRENT PHASE: -${generatePhaseContext()} - -**Response Structure:** -1. **Direct Answer**: Start with clear answer to their question -2. **Simple Explanation**: Explain in non-technical terms -3. **Technical Details** (if relevant): Provide deeper technical context -4. **Recommendation**: Specific action they should take -5. **Why It Matters**: Connect to their goals - -**Example Responses:** - -User: "Which model should I use?" -You: "Based on your dataset characteristics (check the DATA ANALYSIS CONTEXT above for specific numbers), I recommend **LightGBM** as your primary choice. Here's why: - -**Simple explanation:** Think of LightGBM as having thousands of simple decision trees working together to make predictions. It's like asking 1000 experts for their opinion and taking the most popular answer. - -**Why it's best for YOUR data:** -- Handles your numeric and categorical features efficiently -- Naturally deals with missing data (no preprocessing needed!) -- Fast training on your dataset size -- Expected accuracy: 89-95% - -**Technical details:** LightGBM uses leaf-wise tree growth with gradient boosting, providing faster training than level-wise approaches while maintaining high accuracy. - -**Also consider:** XGBoost if maximum accuracy is critical and training time is less important." - -Be conversational, helpful, and always tie recommendations back to the user's specific data characteristics shown in the DATA ANALYSIS CONTEXT above.`; + messagesEndRef.current?.scrollIntoView({ behavior: "smooth" }); + }, [messages, isLoading]); const sendMessage = async () => { - if (!input.trim() || isLoading) return; + const content = input.trim(); + if (!content || isLoading || status !== "active") return; - const userMessage: Message = { role: 'user', content: input }; - setMessages(prev => [...prev, userMessage]); - setInput(''); - setIsLoading(true); + const token = getStoredSessionToken(); + if (!token) return; - // Add empty assistant message that will be filled with streaming content - const assistantMessageIndex = messages.length + 1; - setMessages(prev => [...prev, { role: 'assistant', content: '' }]); + const userMessage: Message = { role: "user", content }; + const history = [...messages, userMessage]; + setMessages(history); + setInput(""); + setIsLoading(true); try { - const requestBody = { - contents: [ - { - role: 'user', - parts: [{ text: systemPrompt }] - }, - { - role: 'model', - parts: [{ text: 'Understood. I am ready to assist with data science questions based on the provided analysis context. I will only reference information from the given data and will clearly state when I cannot answer a question.' }] - }, - ...messages.slice(1).map(msg => ({ - role: msg.role === 'user' ? 'user' : 'model', - parts: [{ text: msg.content }] - })), - { - role: 'user', - parts: [{ text: input }] - } - ], - generationConfig: { - temperature: 0.7, - topK: 40, - topP: 0.95, - maxOutputTokens: 1024, - } - }; + const [datasets, runs] = await Promise.all([ + workspaceDatasetAPI.list().catch(() => []), + workspaceTrainingAPI.list().catch(() => []), + ]); + const latestRun = runs[0] || null; + const systemPrompt = `You are NoCodeML's Data Science Assistant for a temporary, account-free ML workspace. + +Rules: +- Never invent dataset facts, metrics, results or predictions. +- Never ask the user to paste secrets or private credentials. +- The supplied context deliberately excludes raw dataset rows. Do not claim you inspected raw records. +- Explain in plain English first, then concise technical detail. +- Prefer actionable guidance tied to the current workspace. +- Keep normal answers under 250 words unless the user asks for detail. + +Temporary datasets metadata: +${safeJson(datasets.map((dataset) => ({ + name: dataset.name, + row_count: dataset.row_count, + column_count: dataset.column_count, + file_size_bytes: dataset.file_size_bytes, +})))} + +Latest temporary training run: +${safeJson(latestRun ? { + dataset_name: latestRun.dataset_name, + status: latestRun.status, + config: latestRun.config, + results: latestRun.results, + best_model: latestRun.best_model, + error: latestRun.error, +} : null)} +`; - const response = await fetch(GEMINI_API_URL, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(requestBody) + const response = await fetch(`${API_BASE_URL}/api/v1/assistant/chat`, { + method: "POST", + headers: { + "Content-Type": "application/json", + "X-NoCodeML-Session": token, + }, + body: JSON.stringify({ + system_prompt: systemPrompt, + messages: history.slice(-20), + }), }); + const data = await response.json().catch(() => ({})); if (!response.ok) { - throw new Error(`API Error: ${response.status}`); + const detail = data?.detail; + const message = typeof detail === "string" ? detail : detail?.message; + throw new Error(message || "The assistant is temporarily unavailable."); } - const reader = response.body?.getReader(); - const decoder = new TextDecoder(); - let accumulatedText = ''; - - if (reader) { - while (true) { - const { done, value } = await reader.read(); - if (done) break; - - const chunk = decoder.decode(value); - const lines = chunk.split('\n'); - - for (const line of lines) { - if (line.startsWith('data: ')) { - try { - const jsonStr = line.slice(6); // Remove 'data: ' prefix - const data = JSON.parse(jsonStr); - - if (data.candidates && data.candidates[0]?.content?.parts?.[0]?.text) { - const newText = data.candidates[0].content.parts[0].text; - accumulatedText += newText; - - // Update the assistant message with streaming content - setMessages(prev => { - const updated = [...prev]; - updated[assistantMessageIndex] = { - role: 'assistant', - content: accumulatedText - }; - return updated; - }); - - // Add delay for slower, more readable streaming (30ms per chunk) - await new Promise(resolve => setTimeout(resolve, 30)); - } - } catch (e) { - // Skip invalid JSON lines - continue; - } - } - } - } - } - } catch (error: any) { - console.error('Chat error:', error); - setMessages(prev => { - const updated = [...prev]; - updated[assistantMessageIndex] = { - role: 'assistant', - content: 'โŒ Sorry, I encountered an error. Please try again or check the console for details.' - }; - return updated; - }); + setMessages((previous) => [...previous, { role: "assistant", content: data.content || "I could not generate a response." }]); + } catch (error) { + setMessages((previous) => [ + ...previous, + { + role: "assistant", + content: `I couldn't answer that right now. ${error instanceof Error ? error.message : "Please try again."}`, + }, + ]); } finally { setIsLoading(false); } }; - const handleMouseDown = (e: React.MouseEvent) => { - e.preventDefault(); - e.stopPropagation(); - - const buttonRect = buttonRef.current?.getBoundingClientRect(); - if (!buttonRect) return; - - setIsDragging(true); - setDragStart({ - x: e.clientX - buttonRect.left, - y: e.clientY - buttonRect.top - }); - }; - - const handleMouseMove = (e: MouseEvent) => { - if (!isDragging) return; - - e.preventDefault(); - - // Calculate new position based on cursor position - const newX = e.clientX - dragStart.x; - const newY = e.clientY - dragStart.y; - - // Constrain within viewport - const maxX = window.innerWidth - 60; - const maxY = window.innerHeight - 60; - - setPosition({ - x: Math.max(0, Math.min(newX, maxX)), - y: Math.max(0, Math.min(newY, maxY)) - }); - }; - - const handleMouseUp = () => { - setIsDragging(false); - }; - - useEffect(() => { - if (isDragging) { - window.addEventListener('mousemove', handleMouseMove); - window.addEventListener('mouseup', handleMouseUp); - return () => { - window.removeEventListener('mousemove', handleMouseMove); - window.removeEventListener('mouseup', handleMouseUp); - }; - } - }, [isDragging, dragStart]); - - // Resize handlers - const handleResizeMouseDown = (e: React.MouseEvent, direction: 'corner-br' | 'corner-bl' | 'corner-tr' | 'corner-tl' | 'bottom' | 'top' | 'right' | 'left') => { - e.preventDefault(); - e.stopPropagation(); - - setIsResizing(true); - setResizeDirection(direction); - setResizeStart({ - x: e.clientX, - y: e.clientY, - width: chatSize.width, - height: chatSize.height - }); - }; - - const handleResizeMouseMove = (e: MouseEvent) => { - if (!isResizing || !resizeDirection) return; - - e.preventDefault(); - - const deltaX = e.clientX - resizeStart.x; - const deltaY = e.clientY - resizeStart.y; - - // Apply constraints based on resize direction - if (resizeDirection === 'corner-br') { - // Bottom-right corner: increase width and height - const newWidth = Math.max(320, Math.min(800, resizeStart.width + deltaX)); - const newHeight = Math.max(400, Math.min(800, resizeStart.height + deltaY)); - setChatSize({ width: newWidth, height: newHeight }); - } else if (resizeDirection === 'corner-bl') { - // Bottom-left corner: decrease width from left, increase height - const newWidth = Math.max(320, Math.min(800, resizeStart.width - deltaX)); - const newHeight = Math.max(400, Math.min(800, resizeStart.height + deltaY)); - setChatSize({ width: newWidth, height: newHeight }); - } else if (resizeDirection === 'corner-tr') { - // Top-right corner: increase width, decrease height from top - const newWidth = Math.max(320, Math.min(800, resizeStart.width + deltaX)); - const newHeight = Math.max(400, Math.min(800, resizeStart.height - deltaY)); - setChatSize({ width: newWidth, height: newHeight }); - } else if (resizeDirection === 'corner-tl') { - // Top-left corner: decrease width from left, decrease height from top - const newWidth = Math.max(320, Math.min(800, resizeStart.width - deltaX)); - const newHeight = Math.max(400, Math.min(800, resizeStart.height - deltaY)); - setChatSize({ width: newWidth, height: newHeight }); - } else if (resizeDirection === 'right') { - // Resize width only (increase from right) - const newWidth = Math.max(320, Math.min(800, resizeStart.width + deltaX)); - setChatSize({ width: newWidth, height: resizeStart.height }); - } else if (resizeDirection === 'left') { - // Resize width only (decrease from left) - const newWidth = Math.max(320, Math.min(800, resizeStart.width - deltaX)); - setChatSize({ width: newWidth, height: resizeStart.height }); - } else if (resizeDirection === 'bottom') { - // Resize height only (increase from bottom) - const newHeight = Math.max(400, Math.min(800, resizeStart.height + deltaY)); - setChatSize({ width: resizeStart.width, height: newHeight }); - } else if (resizeDirection === 'top') { - // Resize height only (decrease from top) - const newHeight = Math.max(400, Math.min(800, resizeStart.height - deltaY)); - setChatSize({ width: resizeStart.width, height: newHeight }); - } - }; - - const handleResizeMouseUp = () => { - setIsResizing(false); - setResizeDirection(null); - }; - - useEffect(() => { - if (isResizing) { - window.addEventListener('mousemove', handleResizeMouseMove); - window.addEventListener('mouseup', handleResizeMouseUp); - return () => { - window.removeEventListener('mousemove', handleResizeMouseMove); - window.removeEventListener('mouseup', handleResizeMouseUp); - }; - } - }, [isResizing, resizeDirection, resizeStart]); - - if (!edaData && currentPhase === 'analysis') { - return null; // Don't show assistant until EDA data is available in analysis phase - } - return ( <> - {/* Floating Button */} -
setIsOpen((value) => !value)} + disabled={status !== "active"} + className="fixed bottom-5 right-5 z-50 h-12 rounded-full border border-primary/30 bg-background/90 px-4 text-foreground shadow-2xl backdrop-blur-xl hover:bg-primary/10 sm:bottom-6 sm:right-6" > - -
+ {isOpen ? : } + AI Assistant + - {/* Chat Window */} {isOpen && ( - - {/* Header */} -
-
-

๐Ÿค– Data Science Assistant

- {/*

Powered by Gemini 2.0 Flash

*/} + +
+
+
+

Data Science Assistant

Derived context only ยท no raw rows

+
-
- {/* Messages */} -
- {messages.map((message, idx) => ( -
-
- {message.role === 'user' ? ( -

{message.content}

- ) : ( -
- {message.content} -
- )} +
+ {messages.map((message, index) => ( +
+ {message.role === "assistant" &&
} +
+ {message.role === "assistant" ?
{message.content}
: message.content}
))} - {isLoading && ( -
-
- -
-
- )} + {isLoading &&
Thinkingโ€ฆ
}
- {/* Input */} -
+
setInput(e.target.value)} - onKeyPress={(e) => e.key === 'Enter' && !isLoading && sendMessage()} - placeholder={`Ask about ${currentPhase}...`} - className="flex-1 rounded-full bg-input border-border focus:ring-primary" + onChange={(event) => setInput(event.target.value)} + onKeyDown={(event) => { if (event.key === "Enter" && !event.shiftKey) { event.preventDefault(); void sendMessage(); } }} + placeholder="Ask about your model or next stepโ€ฆ" disabled={isLoading} + className="rounded-xl" /> -
- - {/* Resize Handles - Corners */} - {/* Bottom-Right Corner */} -
handleResizeMouseDown(e, 'corner-br')} - className="absolute bottom-0 right-0 w-6 h-6 cursor-nwse-resize group z-10" - style={{ touchAction: 'none' }} - > -
-
- - {/* Bottom-Left Corner */} -
handleResizeMouseDown(e, 'corner-bl')} - className="absolute bottom-0 left-0 w-6 h-6 cursor-nesw-resize group z-10" - style={{ touchAction: 'none' }} - > -
-
- - {/* Top-Right Corner */} -
handleResizeMouseDown(e, 'corner-tr')} - className="absolute top-0 right-0 w-6 h-6 cursor-nesw-resize group z-10" - style={{ touchAction: 'none' }} - > -
-
- - {/* Top-Left Corner */} -
handleResizeMouseDown(e, 'corner-tl')} - className="absolute top-0 left-0 w-6 h-6 cursor-nwse-resize group z-10" - style={{ touchAction: 'none' }} - > -
-
- - {/* Resize Handles - Edges */} - {/* Bottom Edge */} -
handleResizeMouseDown(e, 'bottom')} - className="absolute bottom-0 left-6 right-6 h-3 cursor-ns-resize hover:bg-primary/10 transition-colors z-[5]" - style={{ touchAction: 'none' }} - /> - - {/* Top Edge */} -
handleResizeMouseDown(e, 'top')} - className="absolute top-0 left-6 right-6 h-3 cursor-ns-resize hover:bg-primary/10 transition-colors z-[5]" - style={{ touchAction: 'none' }} - /> - - {/* Right Edge */} -
handleResizeMouseDown(e, 'right')} - className="absolute top-6 bottom-6 right-0 w-3 cursor-ew-resize hover:bg-primary/10 transition-colors z-[5]" - style={{ touchAction: 'none' }} - /> - - {/* Left Edge */} -
handleResizeMouseDown(e, 'left')} - className="absolute top-6 bottom-6 left-0 w-3 cursor-ew-resize hover:bg-primary/10 transition-colors z-[5]" - style={{ touchAction: 'none' }} - /> )} ); }; + +export default DataScienceAssistant; diff --git a/Frontend/src/components/experiments/ExperimentCard.tsx b/Frontend/src/components/experiments/ExperimentCard.tsx index 98d1ce9..b280b6f 100644 --- a/Frontend/src/components/experiments/ExperimentCard.tsx +++ b/Frontend/src/components/experiments/ExperimentCard.tsx @@ -1,4 +1,14 @@ -import { Beaker, Calendar, Trash2, Copy, FolderOpen, MoreVertical } from "lucide-react"; +import { + Beaker, + Calendar, + Copy, + Database, + FolderOpen, + MoreVertical, + Trash2, +} from "lucide-react"; +import { useNavigate } from "react-router-dom"; + import { Button } from "@/components/ui/button"; import { DropdownMenu, @@ -6,7 +16,6 @@ import { DropdownMenuItem, DropdownMenuTrigger, } from "@/components/ui/dropdown-menu"; -import { useNavigate } from "react-router-dom"; interface ExperimentCardProps { experiment: { @@ -26,85 +35,94 @@ interface ExperimentCardProps { const ExperimentCard = ({ experiment, onDelete, onDuplicate }: ExperimentCardProps) => { const navigate = useNavigate(); - const formatDate = (dateString: string) => { - const date = new Date(dateString); - return date.toLocaleDateString('en-US', { - month: 'short', - day: 'numeric', - year: 'numeric' - }); - }; + const formattedDate = new Date(experiment.updatedAt).toLocaleDateString(undefined, { + month: "short", + day: "numeric", + year: "numeric", + }); return ( -
-
-
-
- +
+
+ +
+
+
+
- -
-
-

{experiment.name}

- - {experiment.status === "completed" ? "Completed" : "In Progress"} + +
+
+

{experiment.name}

+ + {experiment.status === "completed" ? "Completed" : "In progress"}
- -
- - - {formatDate(experiment.updatedAt)} + +
+ + + {formattedDate} {experiment.datasetName && ( - {experiment.datasetName} + + + {experiment.datasetName} + )} {experiment.config?.taskType && ( - + {experiment.config.taskType} )}
- -
- - + - - + onDuplicate(experiment.id)}> - + Duplicate - onDelete(experiment.id)} - className="text-destructive" + className="text-destructive focus:text-destructive" > - + Delete
-
+
); }; diff --git a/Frontend/src/components/playground/DataAnalysisStep.tsx b/Frontend/src/components/playground/DataAnalysisStep.tsx index b34da53..deb162a 100644 --- a/Frontend/src/components/playground/DataAnalysisStep.tsx +++ b/Frontend/src/components/playground/DataAnalysisStep.tsx @@ -1,4 +1,4 @@ -import { useEffect, useState } from "react"; +import { useState } from "react"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; import { Button } from "@/components/ui/button"; @@ -10,6 +10,7 @@ import { useEDA } from "@/hooks/useEDA"; import { DatasetOverview } from "./DatasetOverview"; import { DetailedAnalysis } from "./DetailedAnalysis"; import { VisualizationPanel } from "./VisualizationPanel"; +import DataReadinessPanel from "./DataReadinessPanel"; import { toast } from "sonner"; interface DataAnalysisStepProps { @@ -19,24 +20,22 @@ interface DataAnalysisStepProps { export const DataAnalysisStep = ({ onNext }: DataAnalysisStepProps) => { const { currentExperiment } = useExperiment(); const datasetId = currentExperiment?.datasetId; - + const { edaData, isLoading, error, loadEDASummary } = useEDA(datasetId); const [activeTab, setActiveTab] = useState("overview"); const [retryCount, setRetryCount] = useState(0); - // Handle retry with exponential backoff const handleRetry = () => { if (retryCount < 3) { setRetryCount(retryCount + 1); setTimeout(() => { - loadEDASummary(true); // Pass true to force refresh - }, Math.pow(2, retryCount) * 1000); // 1s, 2s, 4s + loadEDASummary(true); + }, Math.pow(2, retryCount) * 1000); } else { toast.error("Maximum retry attempts reached. Please refresh the page."); } }; - // Loading state if (isLoading && !edaData) { return (
@@ -60,7 +59,6 @@ export const DataAnalysisStep = ({ onNext }: DataAnalysisStepProps) => { ); } - // Error state if (error) { return (
@@ -69,7 +67,7 @@ export const DataAnalysisStep = ({ onNext }: DataAnalysisStepProps) => { Failed to Load Data Analysis {error} -
+
@@ -83,18 +81,15 @@ export const DataAnalysisStep = ({ onNext }: DataAnalysisStepProps) => { ); } - // No data state if (!edaData) { return (
-
+

No data available

-

- Please select a dataset to analyze -

+

Please select a dataset to analyze

@@ -104,20 +99,24 @@ export const DataAnalysisStep = ({ onNext }: DataAnalysisStepProps) => { return (
- + + + Data Analysis

- Explore your dataset with statistics, visualizations, and detailed column analysis + Explore statistics, visualizations, relationships and detailed column information before training.

- - Overview - Detailed Analysis - Visualizations - +
+ + Overview + Detailed Analysis + Visualizations + +
{ { - {/* Navigation Buttons */} -
- {onNext && ( diff --git a/Frontend/src/components/playground/DataReadinessPanel.tsx b/Frontend/src/components/playground/DataReadinessPanel.tsx new file mode 100644 index 0000000..9d1ebcd --- /dev/null +++ b/Frontend/src/components/playground/DataReadinessPanel.tsx @@ -0,0 +1,177 @@ +import { AlertTriangle, CheckCircle2, Database, Sparkles, Target } from "lucide-react"; + +import { Badge } from "@/components/ui/badge"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import type { EDAResponse } from "@/types/experiment"; + +interface DataReadinessPanelProps { + edaData: EDAResponse; +} + +type TaskType = "classification" | "regression"; + +const clamp = (value: number, min: number, max: number) => Math.min(max, Math.max(min, value)); + +const DataReadinessPanel = ({ edaData }: DataReadinessPanelProps) => { + const rows = edaData.dataset_info.row_count || 0; + const columns = edaData.columns || []; + const missingPercent = edaData.missing_data_summary?.missing_percent || 0; + const constantColumns = columns.filter((column) => column.unique_count <= 1); + const highMissingColumns = columns.filter((column) => column.missing_percent >= 40); + const categoricalSet = new Set(edaData.categorical_columns || []); + const numericSet = new Set(edaData.numeric_columns || []); + const idSet = new Set(edaData.id_columns || []); + const highCardinalityCategoricals = columns.filter( + (column) => + categoricalSet.has(column.name) && + !idSet.has(column.name) && + column.unique_count > Math.max(50, Math.floor(rows * 0.5)), + ); + + const sizePenalty = rows < 50 ? 25 : rows < 200 ? 12 : rows < 500 ? 5 : 0; + const missingPenalty = Math.min(35, missingPercent * 0.55); + const constantPenalty = Math.min(18, constantColumns.length * 6); + const highMissingPenalty = Math.min(15, highMissingColumns.length * 4); + const cardinalityPenalty = Math.min(10, highCardinalityCategoricals.length * 3); + const score = Math.round( + clamp(100 - sizePenalty - missingPenalty - constantPenalty - highMissingPenalty - cardinalityPenalty, 0, 100), + ); + + const readiness = + score >= 85 + ? { label: "Ready", detail: "Strong starting point for model training." } + : score >= 70 + ? { label: "Good", detail: "Usable now, with a few things worth reviewing." } + : score >= 50 + ? { label: "Needs review", detail: "Training is possible, but data quality may limit results." } + : { label: "High risk", detail: "Resolve the highlighted data issues before trusting model results." }; + + const classificationThreshold = Math.max(20, Math.floor(rows * 0.05)); + const targetHints = [ + "target", + "label", + "outcome", + "class", + "churn", + "survived", + "fraud", + "default", + "price", + "sales", + "revenue", + "score", + "rating", + ]; + + const targetCandidates = columns + .filter( + (column) => + !idSet.has(column.name) && + column.unique_count > 1 && + column.missing_percent < 50, + ) + .map((column, index) => { + const lowerName = column.name.toLowerCase(); + const nameHint = targetHints.some((hint) => lowerName === hint || lowerName.includes(hint)); + const numericLowCardinality = + numericSet.has(column.name) && column.unique_count <= classificationThreshold; + const task: TaskType = + categoricalSet.has(column.name) || numericLowCardinality ? "classification" : "regression"; + let candidateScore = nameHint ? 12 : 0; + candidateScore += column.missing_percent === 0 ? 3 : column.missing_percent < 10 ? 2 : 0; + candidateScore += categoricalSet.has(column.name) && column.unique_count <= 20 ? 4 : 0; + candidateScore += numericLowCardinality ? 3 : 0; + candidateScore += index === columns.length - 1 ? 2 : 0; + return { column: column.name, task, score: candidateScore }; + }) + .sort((a, b) => b.score - a.score) + .slice(0, 3); + + const recommendations: string[] = []; + if (rows < 200) recommendations.push("The dataset is small; prefer simpler models and treat test metrics cautiously."); + if (missingPercent >= 10) recommendations.push(`${missingPercent.toFixed(1)}% of cells are missing; review imputation before interpreting results.`); + if (highMissingColumns.length) recommendations.push(`${highMissingColumns.length} column(s) are at least 40% missing and may be better excluded.`); + if (constantColumns.length) recommendations.push(`${constantColumns.length} constant column(s) contain no predictive signal.`); + if (highCardinalityCategoricals.length) recommendations.push(`${highCardinalityCategoricals.length} categorical column(s) have very high cardinality and may overfit.`); + if (idSet.size) recommendations.push(`${idSet.size} likely ID column(s) were detected and will be excluded from recommended features.`); + if (!recommendations.length) recommendations.push("No major readiness issues were detected. You can move on to model configuration."); + + return ( + + +
+
+
+ Smart data check +
+ ML Readiness +

A transparent pre-training check based on size, missingness, cardinality and unusable columns.

+
+
+
{score}
+
+ = 70 ? "secondary" : "outline"}>{readiness.label} +
out of 100
+
+
+
+ + + +
+
+
Rows
+
{rows.toLocaleString()}
+
+
+
Missing cells
+
{missingPercent.toFixed(1)}%
+
+
+
Likely IDs
+
{idSet.size}
+
+
+
Risky columns
+
{new Set([...constantColumns, ...highMissingColumns, ...highCardinalityCategoricals].map((column) => column.name)).size}
+
+
+ +
+
+
+ {score >= 70 ? : } + What to review +
+
    + {recommendations.slice(0, 4).map((recommendation) => ( +
  • โ€ข{recommendation}
    • + ))} +
+
+ +
+
Possible prediction targets
+ {targetCandidates.length ? ( +
+ {targetCandidates.map((candidate) => ( +
+ {candidate.column} + {candidate.task} +
+ ))} +

Suggestions are heuristic. You still choose the business/ML target in the next step.

+
+ ) : ( +

No reliable target suggestion was found automatically. Choose the target manually in Model Configuration.

+ )} +
+
+ +

{readiness.detail} This score is guidance, not a substitute for domain knowledge or validation on unseen data.

+ + + ); +}; + +export default DataReadinessPanel; diff --git a/Frontend/src/components/playground/ModelConfigStep.tsx b/Frontend/src/components/playground/ModelConfigStep.tsx index d40da54..e0731e6 100644 --- a/Frontend/src/components/playground/ModelConfigStep.tsx +++ b/Frontend/src/components/playground/ModelConfigStep.tsx @@ -1,688 +1,614 @@ -import { useState, useEffect, useRef, useCallback, useMemo } from "react"; -import { Settings, Target, AlertCircle, CheckCircle2, Loader2, Sparkles, TrendingUp, Zap, Info } from "lucide-react"; +import { useEffect, useMemo, useState } from "react"; +import { + AlertCircle, + CheckCircle2, + Info, + Loader2, + Settings, + Sparkles, + Target, + TrendingUp, + Zap, +} from "lucide-react"; + +import { Alert, AlertDescription } from "@/components/ui/alert"; +import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; +import { Checkbox } from "@/components/ui/checkbox"; import { Label } from "@/components/ui/label"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; -import { Checkbox } from "@/components/ui/checkbox"; import { Slider } from "@/components/ui/slider"; -import { FeatureSelector } from "./FeatureSelector"; -import { experimentAPI } from "@/services/apiService"; -import { useToast } from "@/hooks/use-toast"; -import { Alert, AlertDescription } from "@/components/ui/alert"; -import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; import { Switch } from "@/components/ui/switch"; -import { ExperimentConfig, ModelConfig, ColumnInfo, EDAResponse } from '@/types/experiment'; -import { useModels } from "@/contexts/ModelsContext"; import { useExperiment } from "@/contexts/ExperimentContext"; +import { useModels } from "@/contexts/ModelsContext"; +import { useToast } from "@/hooks/use-toast"; +import type { ColumnInfo, EDAResponse } from "@/types/experiment"; + +import { FeatureSelector } from "./FeatureSelector"; + +type TaskType = "classification" | "regression"; interface ModelConfigStepProps { experiment: any; - edaData: any; + edaData: EDAResponse; onNext: () => void; onBack?: () => void; } +interface TargetSuggestion { + column: string; + task: TaskType; + score: number; + reason: string; +} + +const TARGET_HINTS = [ + "target", + "label", + "outcome", + "class", + "churn", + "survived", + "fraud", + "default", + "price", + "sales", + "revenue", + "score", + "rating", +]; + const ModelConfigStep = ({ experiment, edaData, onNext, onBack }: ModelConfigStepProps) => { const { toast } = useToast(); const { updateExperiment } = useExperiment(); const { getModelsByTask, loading: modelsLoading } = useModels(); - const [isSaving, setIsSaving] = useState(false); + + const [taskType, setTaskType] = useState(experiment?.config?.taskType || "classification"); + const [targetColumn, setTargetColumn] = useState(experiment?.config?.targetColumn || ""); + const [selectedFeatures, setSelectedFeatures] = useState(experiment?.config?.selectedFeatures || []); + const [trainSplit, setTrainSplit] = useState([ + experiment?.config?.trainTestSplit ? experiment.config.trainTestSplit * 100 : 80, + ]); + const [selectedModels, setSelectedModels] = useState( + experiment?.config?.models?.map((model: any) => model.model_type) || [], + ); + const [optimizationEnabled, setOptimizationEnabled] = useState(Boolean(experiment?.config?.enableOptimization)); const [validationErrors, setValidationErrors] = useState([]); + const [isSaving, setIsSaving] = useState(false); const [hasUnsavedChanges, setHasUnsavedChanges] = useState(false); - - // Config state - const [taskType, setTaskType] = useState<"classification" | "regression">( - experiment?.config?.taskType || "classification" + + const rows = edaData?.dataset_info?.row_count || 0; + const allColumns = useMemo(() => edaData?.columns || [], [edaData?.columns]); + const idColumns = useMemo(() => edaData?.id_columns || [], [edaData?.id_columns]); + const idSet = useMemo(() => new Set(idColumns), [idColumns]); + const numericColumns = useMemo( + () => (edaData?.numeric_columns || []).filter((column) => !idSet.has(column)), + [edaData?.numeric_columns, idSet], ); - const [targetColumn, setTargetColumn] = useState( - experiment?.config?.targetColumn || "" + const categoricalColumns = useMemo( + () => (edaData?.categorical_columns || []).filter((column) => !idSet.has(column)), + [edaData?.categorical_columns, idSet], ); - const [selectedFeatures, setSelectedFeatures] = useState( - experiment?.config?.selectedFeatures || [] + const availableColumns = useMemo( + () => Array.from(new Set([...numericColumns, ...categoricalColumns])), + [numericColumns, categoricalColumns], ); - const [trainSplit, setTrainSplit] = useState([ - experiment?.config?.trainTestSplit ? experiment.config.trainTestSplit * 100 : 80 - ]); - const [selectedModels, setSelectedModels] = useState([]); - const [optimizationEnabled, setOptimizationEnabled] = useState( - experiment?.config?.enableOptimization || false + const columnMap = useMemo( + () => new Map(allColumns.map((column) => [column.name, column])), + [allColumns], ); - - // Get available columns from EDA (memoized to prevent re-renders) - const allColumns = useMemo(() => edaData?.columns || [], [edaData?.columns]); - const numericColumns = useMemo(() => - edaData?.numeric_columns?.filter( - (col: string) => !edaData.id_columns.includes(col) - ) || [] - , [edaData?.numeric_columns, edaData?.id_columns]); - - const categoricalColumns = useMemo(() => - edaData?.categorical_columns?.filter( - (col: string) => !edaData.id_columns.includes(col) - ) || [] - , [edaData?.categorical_columns, edaData?.id_columns]); - - const idColumns = useMemo(() => edaData?.id_columns || [], [edaData?.id_columns]); - const availableColumns = useMemo(() => - [...numericColumns, ...categoricalColumns] - , [numericColumns, categoricalColumns]); - - // Valid target columns based on task type - const validTargetColumns = useMemo(() => { - if (taskType === "classification") { - // Classification: only categorical columns - return categoricalColumns; - } else { - // Regression: only numerical columns - return numericColumns; - } - }, [taskType, categoricalColumns, numericColumns]); - - // Available features (exclude target column and ID columns) - const availableFeatures = useMemo(() => { - return availableColumns.filter(col => col !== targetColumn); - }, [availableColumns, targetColumn]); - - // Get models from the global context based on task type - const models = useMemo(() => { - const modelsData = getModelsByTask(taskType); - return modelsData.map(model => ({ - id: model.model_type, - name: model.display_name, - description: model.description - })); - }, [taskType, getModelsByTask]); - - // Sync state with experiment config when it changes (e.g., after save or tab switch) - useEffect(() => { - if (experiment?.config) { - // Update task type if different - if (experiment.config.taskType && experiment.config.taskType !== taskType) { - setTaskType(experiment.config.taskType); - } - - // Update target column if different - if (experiment.config.targetColumn && experiment.config.targetColumn !== targetColumn) { - setTargetColumn(experiment.config.targetColumn); - } - - // Update selected features if different - if (experiment.config.selectedFeatures && experiment.config.selectedFeatures.length > 0) { - setSelectedFeatures(experiment.config.selectedFeatures); - } else if (selectedFeatures.length === 0 && availableColumns.length > 0) { - // Auto-select all non-ID columns by default if no features selected yet - setSelectedFeatures(availableColumns); - } - - // Update train split if different - if (experiment.config.trainTestSplit && experiment.config.trainTestSplit !== trainSplit[0] / 100) { - setTrainSplit([experiment.config.trainTestSplit * 100]); - } - - // Update selected models if different - if (experiment.config.models && experiment.config.models.length > 0) { - const modelTypes = experiment.config.models.map((m: any) => m.model_type); - if (JSON.stringify(modelTypes) !== JSON.stringify(selectedModels)) { - setSelectedModels(modelTypes); + const numericSet = useMemo(() => new Set(numericColumns), [numericColumns]); + const categoricalSet = useMemo(() => new Set(categoricalColumns), [categoricalColumns]); + const classificationCardinalityLimit = Math.max(20, Math.floor(rows * 0.05)); + + const inferTask = (column: string): TaskType => { + const info = columnMap.get(column); + if (categoricalSet.has(column)) return "classification"; + if (numericSet.has(column) && info && info.unique_count <= classificationCardinalityLimit) return "classification"; + return "regression"; + }; + + const classificationTargets = useMemo( + () => + availableColumns.filter((column) => { + const info = columnMap.get(column); + if (!info || info.unique_count < 2) return false; + return categoricalSet.has(column) || (numericSet.has(column) && info.unique_count <= classificationCardinalityLimit); + }), + [availableColumns, categoricalSet, classificationCardinalityLimit, columnMap, numericSet], + ); + + const validTargetColumns = useMemo( + () => (taskType === "classification" ? classificationTargets : numericColumns), + [classificationTargets, numericColumns, taskType], + ); + + const availableFeatures = useMemo( + () => availableColumns.filter((column) => column !== targetColumn), + [availableColumns, targetColumn], + ); + + const models = useMemo( + () => + getModelsByTask(taskType).map((model) => ({ + id: model.model_type, + name: model.display_name, + description: model.description, + })), + [getModelsByTask, taskType], + ); + + const targetSuggestions = useMemo(() => { + return allColumns + .filter( + (column) => + !idSet.has(column.name) && + column.unique_count > 1 && + column.missing_percent < 50 && + (numericSet.has(column.name) || categoricalSet.has(column.name)), + ) + .map((column, index) => { + const lowerName = column.name.toLowerCase(); + const hasNameHint = TARGET_HINTS.some((hint) => lowerName === hint || lowerName.includes(hint)); + const inferredTask = inferTask(column.name); + let score = 0; + const reasons: string[] = []; + + if (hasNameHint) { + score += 12; + reasons.push("target-like name"); } - } - } - }, [experiment?.config, availableColumns.length]); + if (column.missing_percent === 0) { + score += 3; + reasons.push("no missing values"); + } else if (column.missing_percent < 10) { + score += 2; + } + if (inferredTask === "classification" && column.unique_count <= 20) { + score += 4; + reasons.push(`${column.unique_count} classes`); + } + if (index === allColumns.length - 1) { + score += 2; + reasons.push("final dataset column"); + } + + return { + column: column.name, + task: inferredTask, + score, + reason: reasons.slice(0, 2).join(" ยท ") || "usable target candidate", + }; + }) + .sort((a, b) => b.score - a.score) + .slice(0, 3); + }, [allColumns, categoricalSet, idSet, numericSet]); + + useEffect(() => { + if (hasUnsavedChanges || !experiment?.config) return; + const config = experiment.config; + setTaskType(config.taskType || "classification"); + setTargetColumn(config.targetColumn || ""); + setSelectedFeatures(config.selectedFeatures || []); + setTrainSplit([config.trainTestSplit ? config.trainTestSplit * 100 : 80]); + setSelectedModels(config.models?.map((model: any) => model.model_type) || []); + setOptimizationEnabled(Boolean(config.enableOptimization)); + }, [experiment?.config, hasUnsavedChanges]); - // Clear invalid models and target when task type changes - const prevTaskType = useRef(taskType); useEffect(() => { - if (prevTaskType.current !== taskType && prevTaskType.current !== undefined) { - // Clear invalid models - const availableModelIds = models.map(m => m.id); - const validModels = selectedModels.filter(modelId => - availableModelIds.includes(modelId) + if (!experiment?.config?.selectedFeatures?.length && !hasUnsavedChanges && !selectedFeatures.length) { + setSelectedFeatures( + availableColumns.filter((column) => { + const info = columnMap.get(column); + return info && info.unique_count > 1 && info.missing_percent < 50; + }), ); - - if (validModels.length !== selectedModels.length) { - setSelectedModels(validModels); - - if (selectedModels.length > 0 && validModels.length === 0) { - toast({ - title: "Model Selection Cleared", - description: `Previous model selections were cleared because they're not available for ${taskType}.`, - }); - } - } + } + }, [availableColumns, columnMap, experiment?.config?.selectedFeatures, hasUnsavedChanges, selectedFeatures.length]); - // Clear target if it's not valid for new task type - if (targetColumn && !validTargetColumns.includes(targetColumn)) { - setTargetColumn(""); - toast({ - title: "Target Column Cleared", - description: `The previous target column is not valid for ${taskType}. Please select a ${taskType === "classification" ? "categorical" : "numerical"} column.`, - }); - } + const markChanged = () => { + setHasUnsavedChanges(true); + if (validationErrors.length) setValidationErrors([]); + }; + + const handleTaskChange = (nextTask: TaskType) => { + setTaskType(nextTask); + const validTargets = nextTask === "classification" ? classificationTargets : numericColumns; + if (targetColumn && !validTargets.includes(targetColumn)) { + setTargetColumn(""); + setSelectedFeatures((current) => current.filter((feature) => feature !== targetColumn)); } - prevTaskType.current = taskType; - }, [taskType, models, selectedModels, validTargetColumns, targetColumn, toast]); - - // Handle target column change + setSelectedModels([]); + markChanged(); + }; + const handleTargetChange = (value: string) => { + const inferredTask = inferTask(value); setTargetColumn(value); - - // Automatically remove target from selected features - if (selectedFeatures.includes(value)) { - setSelectedFeatures(prev => prev.filter(f => f !== value)); + setSelectedFeatures((current) => current.filter((feature) => feature !== value)); + if (inferredTask !== taskType) { + setTaskType(inferredTask); + setSelectedModels([]); toast({ - title: "Feature Removed", - description: "Target column was automatically removed from selected features.", + title: "Task type inferred", + description: `${value} looks like a ${inferredTask} target. You can still change the task manually.`, }); } + markChanged(); }; - const toggleModel = (modelId: string) => { - setSelectedModels(prev => - prev.includes(modelId) - ? prev.filter(id => id !== modelId) - : [...prev, modelId] - ); + const recommendedFeaturesFor = (target: string) => { + const recommended = availableColumns.filter((column) => { + if (column === target || idSet.has(column)) return false; + const info = columnMap.get(column); + if (!info || info.unique_count <= 1 || info.missing_percent >= 50) return false; + if (categoricalSet.has(column)) { + return info.unique_count <= Math.max(100, Math.floor(rows * 0.25)); + } + return true; + }); + + return recommended.length + ? recommended + : availableColumns.filter((column) => column !== target && !idSet.has(column)); }; - - const handleFeatureToggle = (feature: string) => { - // Prevent selecting target as feature - if (feature === targetColumn) { + + const applySmartSetup = (preferredTarget?: string) => { + const target = preferredTarget || targetColumn || targetSuggestions[0]?.column; + if (!target) { toast({ - title: "Invalid Selection", - description: "Target column cannot be used as a feature.", - variant: "destructive" + title: "Choose a target first", + description: "No reliable target candidate was found automatically.", + variant: "destructive", }); return; } - setSelectedFeatures(prev => - prev.includes(feature) - ? prev.filter(f => f !== feature) - : [...prev, feature] - ); + const inferredTask = inferTask(target); + const smartFeatures = recommendedFeaturesFor(target); + const taskModels = getModelsByTask(inferredTask); + const modelIds = taskModels.map((model) => model.model_type).slice(0, 4); + const split = rows < 300 ? 75 : rows > 10000 ? 85 : 80; + + setTargetColumn(target); + setTaskType(inferredTask); + setSelectedFeatures(smartFeatures); + setSelectedModels(modelIds); + setTrainSplit([split]); + setOptimizationEnabled(true); + markChanged(); + + toast({ + title: "Smart AutoML setup applied", + description: `${inferredTask} ยท ${smartFeatures.length} features ยท ${modelIds.length} models ยท ${split}/${100 - split} split`, + }); }; - - const handleSelectAllFeatures = () => { - // Exclude target column from selection - setSelectedFeatures(availableFeatures); + + const toggleModel = (modelId: string) => { + setSelectedModels((current) => + current.includes(modelId) ? current.filter((id) => id !== modelId) : [...current, modelId], + ); + markChanged(); }; - - const handleClearAllFeatures = () => { - setSelectedFeatures([]); + + const handleFeatureToggle = (feature: string) => { + if (feature === targetColumn) { + toast({ title: "Invalid selection", description: "The target cannot also be a feature.", variant: "destructive" }); + return; + } + setSelectedFeatures((current) => + current.includes(feature) ? current.filter((item) => item !== feature) : [...current, feature], + ); + markChanged(); }; const handleSelectRecommended = () => { - const recommended: string[] = []; - - // All numeric columns (excluding target) - numericColumns.forEach(col => { - if (col !== targetColumn) { - recommended.push(col); - } - }); - - // Categorical with reasonable cardinality and low missing (excluding target) - categoricalColumns.forEach(col => { - if (col !== targetColumn) { - const info = allColumns.find((c: any) => c.name === col); - if (info && info.unique_count <= 50 && info.missing_percent < 50) { - recommended.push(col); - } - } - }); - - setSelectedFeatures(recommended); + setSelectedFeatures(recommendedFeaturesFor(targetColumn)); + markChanged(); }; - - // Track unsaved changes - useEffect(() => { - setHasUnsavedChanges(true); - }, [taskType, targetColumn, selectedFeatures.length, trainSplit, selectedModels.length]); - - // Clear validation errors when user makes changes - useEffect(() => { - if (validationErrors.length > 0) { - setValidationErrors([]); - } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [taskType, targetColumn, selectedFeatures.length, selectedModels.length]); - // Manual save function - const handleSaveConfig = async () => { - // Comprehensive frontend validation + const buildValidationErrors = () => { const errors: string[] = []; - - if (!taskType) { - errors.push("Task type is required"); - } - - if (!targetColumn) { - errors.push("Target column is required"); - } else { - // Validate target column is appropriate for task type - if (taskType === "classification" && !categoricalColumns.includes(targetColumn)) { - errors.push("For classification, target must be a categorical column"); - } else if (taskType === "regression" && !numericColumns.includes(targetColumn)) { - errors.push("For regression, target must be a numerical column"); - } - - // Check target is not in features - if (selectedFeatures.includes(targetColumn)) { - errors.push("Target column cannot be a feature"); - } + if (!targetColumn) errors.push("Choose the target column you want to predict."); + if (targetColumn && !validTargetColumns.includes(targetColumn)) { + errors.push(`The selected target is not valid for ${taskType}.`); } - - if (selectedFeatures.length === 0) { - errors.push("Select at least one feature"); - } - - // Check for ID columns in features - const idFeaturesSelected = selectedFeatures.filter(f => idColumns.includes(f)); - if (idFeaturesSelected.length > 0) { - errors.push(`Warning: ID columns detected in features: ${idFeaturesSelected.join(", ")}. Consider removing them.`); - } - - if (selectedModels.length === 0) { - errors.push("Select at least one model"); - } - - if (errors.length > 0) { + if (!selectedFeatures.length) errors.push("Select at least one usable feature."); + if (selectedFeatures.includes(targetColumn)) errors.push("The target column cannot also be a feature."); + const selectedIds = selectedFeatures.filter((feature) => idSet.has(feature)); + if (selectedIds.length) errors.push(`Remove likely ID columns from features: ${selectedIds.join(", ")}.`); + if (!selectedModels.length) errors.push("Select at least one model to train."); + return errors; + }; + + const handleSaveConfig = async (): Promise => { + const errors = buildValidationErrors(); + if (errors.length) { setValidationErrors(errors); - toast({ - title: "Validation Error", - description: "Please fix the errors before saving", - variant: "destructive" - }); - return; + toast({ title: "Configuration needs attention", description: errors[0], variant: "destructive" }); + return false; } - + setValidationErrors([]); setIsSaving(true); - try { - const featureTypes = { - numerical: selectedFeatures.filter(f => numericColumns.includes(f)), - categorical: selectedFeatures.filter(f => categoricalColumns.includes(f)) - }; - - const modelConfigs = selectedModels.map(modelType => { - const modelInfo = models.find(m => m.id === modelType); - return { - model_type: modelType, - display_name: modelInfo?.name || modelType, - preset: "default", // Changed from "balanced" to valid preset - hyperparameters: {}, - custom_hyperparameters: null - }; - }); - + const modelLookup = new Map(getModelsByTask(taskType).map((model) => [model.model_type, model])); const config = { taskType, targetColumn, selectedFeatures, - featureTypes, + featureTypes: { + numerical: selectedFeatures.filter((feature) => numericSet.has(feature)), + categorical: selectedFeatures.filter((feature) => categoricalSet.has(feature)), + }, excludedColumns: idColumns, trainTestSplit: trainSplit[0] / 100, randomSeed: 42, - models: modelConfigs, - enableOptimization: optimizationEnabled + models: selectedModels.map((modelType) => ({ + model_type: modelType, + display_name: modelLookup.get(modelType)?.display_name || modelType, + preset: "default", + hyperparameters: {}, + custom_hyperparameters: null, + })), + enableOptimization: optimizationEnabled, }; - - // Use context to update experiment - this updates both backend AND context state + await updateExperiment(experiment.id, { config }); - setHasUnsavedChanges(false); - toast({ - title: "Success", - description: "Configuration saved successfully" - }); + toast({ title: "Configuration saved", description: "This experiment is ready for training." }); + return true; } catch (error: any) { - console.error("Save failed:", error); - - // Handle different error response structures - const errorDetail = error.response?.data?.detail; - - // Parse validation errors from FastAPI/Pydantic - let errors: string[] = []; - - if (Array.isArray(errorDetail)) { - // Pydantic validation errors format: [{type, loc, msg, input, ctx}, ...] - errors = errorDetail.map(err => { - if (typeof err === 'object' && err.msg) { - const location = err.loc ? ` (${err.loc.join(' -> ')})` : ''; - return `${err.msg}${location}`; - } - return typeof err === 'string' ? err : JSON.stringify(err); - }); - } else if (typeof errorDetail === 'string') { - errors = [errorDetail]; - } else if (typeof errorDetail === 'object' && errorDetail?.message) { - errors = [errorDetail.message]; - } else { - errors = [error.message || "Failed to save configuration"]; - } - - setValidationErrors(errors); - toast({ - title: "Error", - description: errors[0] || "Failed to save configuration", - variant: "destructive" - }); + const detail = error?.response?.data?.detail; + const errorsFromApi = Array.isArray(detail) + ? detail.map((item: any) => item?.msg || String(item)) + : [typeof detail === "string" ? detail : error?.message || "Failed to save configuration."]; + setValidationErrors(errorsFromApi); + toast({ title: "Save failed", description: errorsFromApi[0], variant: "destructive" }); + return false; } finally { setIsSaving(false); } }; const validateAndContinue = async () => { - // Save first, then continue - await handleSaveConfig(); - - // If there are validation errors after save, don't continue - if (validationErrors.length > 0) { + const errors = buildValidationErrors(); + if (errors.length) { + setValidationErrors(errors); return; } - - // Continue to next step + + if (hasUnsavedChanges) { + const saved = await handleSaveConfig(); + if (!saved) return; + } onNext(); }; - + return (
- {/* Validation Errors */} {validationErrors.length > 0 && ( -
    - {validationErrors.map((error, idx) => ( -
  • {error}
    • - ))} +
      + {validationErrors.map((error) =>
    • {error}
      • )}
    )} - - {/* Basic Configuration */} -
    -

    - - Model Configuration -

    - -
    - {/* Task Type */} + + + +
    +
    +
    + Smart AutoML +
    + Configure a strong baseline automatically + + NoCodeML uses transparent dataset heuristics to suggest the task, target, usable features, model comparison set and train/test split. Nothing is lockedโ€”you can edit every choice afterward. + +
    + +
    +     + +
    +
    Rows
    {rows.toLocaleString()}
    +
    Usable columns
    {availableColumns.length}
    +
    Likely IDs excluded
    {idColumns.length}
    +
    Models available
    {models.length}
    +
    + + {targetSuggestions.length > 0 && ( +
    +
    Suggested prediction targets
    +
    + {targetSuggestions.map((suggestion) => ( + + ))} +
    +

    Target suggestions are heuristic. Domain knowledge still wins when the desired outcome is known.

    +
    + )} +     +     + + + + Model Configuration + Fine-tune the prediction task and evaluation split. + +
    - - handleTaskChange(value)}> + Classification Regression +

    Numeric labels such as 0/1 are supported for classification.

    - - {/* Target Variable */} +
    - + - {targetColumn && ( -

    - This is the column you want to predict -

    - )} - {validTargetColumns.length === 0 && ( -

    - โš ๏ธ No valid target columns for {taskType}. Try switching task type. -

    - )}
    - - {/* Train/Test Split */} -
    - + +
    +
    + + {trainSplit[0]}% / {100 - trainSplit[0]}% +
    { setTrainSplit(value); markChanged(); }} min={60} max={90} step={5} - className="mt-2" /> -

    - {trainSplit[0]}% of data will be used for training, {100 - trainSplit[0]}% for testing -

    +

    The test portion remains unseen during model fitting and is used for the reported final metrics.

    -
    -
    - - {/* Feature Selection */} + + + col !== targetColumn)} - categoricalColumns={categoricalColumns.filter(col => col !== targetColumn)} + numericColumns={numericColumns.filter((column) => column !== targetColumn)} + categoricalColumns={categoricalColumns.filter((column) => column !== targetColumn)} idColumns={idColumns} allColumns={allColumns} selectedFeatures={selectedFeatures} onFeatureToggle={handleFeatureToggle} - onSelectAll={handleSelectAllFeatures} - onClearAll={handleClearAllFeatures} + onSelectAll={() => { setSelectedFeatures(availableFeatures); markChanged(); }} + onClearAll={() => { setSelectedFeatures([]); markChanged(); }} onSelectRecommended={handleSelectRecommended} targetColumn={targetColumn} /> - - {/* Model Selection */} -
    -

    Select Models to Train

    - {modelsLoading ? ( -
    - - Loading available models... -
    - ) : models.length === 0 ? ( -
    - - No models available for {taskType} -
    - ) : ( -
    - {models.map(model => ( -
    toggleModel(model.id)} - title={model.description} - > -
    - toggleModel(model.id)} - /> -
    - {model.name} - {model.description && ( -

    - {model.description} -

    - )} + + + + Models to compare + Train multiple algorithms on the same split so the comparison is fair. + + + {modelsLoading ? ( +
    Loading modelsโ€ฆ
    + ) : models.length === 0 ? ( +
    No models are available for this task.
    + ) : ( +
    + {models.map((model) => { + const checked = selectedModels.includes(model.id); + return ( +
    toggleModel(model.id)} + onKeyDown={(event) => { if (event.key === "Enter" || event.key === " ") toggleModel(model.id); }} + className={`cursor-pointer rounded-2xl border p-4 transition ${checked ? "border-primary/60 bg-primary/[0.08]" : "border-border/60 bg-background/25 hover:border-primary/35"}`} + > +
    + event.stopPropagation()} + onCheckedChange={() => toggleModel(model.id)} + aria-label={`Select ${model.name}`} + /> +
    +
    {model.name}
    + {model.description &&

    {model.description}

    } +
    + {checked && } +
    -
    -
    - ))} -
    - )} -

    - {selectedModels.length} model{selectedModels.length !== 1 ? 's' : ''} selected -

    -
    + ); + })} +
    + )} +
    {selectedModels.length} model{selectedModels.length === 1 ? "" : "s"} selected
    + + - {/* Hyperparameter Optimization Card */} - - -
    -
    -
    - -
    + + +
    +
    +
    - - Expert System Optimization - - - Transparent, rule-based parameter tuning with full reasoning visibility - + Expert System Optimization + Dataset-aware, transparent rule-based hyperparameter adjustment.
    - + { setOptimizationEnabled(value); markChanged(); }} />
    - {optimizationEnabled && ( - - + + - - Expert System enabled: Parameters will be adjusted by our - rules engine based on your dataset characteristics, with every decision - explained transparently. - + The optimizer records the rules it applied, so results can explain why parameters changed instead of hiding the process. - -
    -
    - -
    -
    Dataset-Aware
    -
    - Automatically adjusts for dataset size, features, and class imbalance -
    -
    -
    - -
    - -
    -
    Overfitting Prevention
    -
    - Applies regularization and complexity controls based on data -
    -
    -
    - -
    - -
    -
    Research-Backed
    -
    - Uses proven parameters from academic research and competitions -
    -
    -
    -
    - -
    - How it works: Our transparent Expert System uses research-backed - heuristic rules to analyze your dataset's characteristics (size, features, class balance) - and adjusts each hyperparameter with a clear, step-by-step explanation of every decision made. +
    +
    Dataset-aware
    Adapts to sample count, dimensionality and imbalance.
    +
    Overfit controls
    Adjusts complexity and regularization where appropriate.
    +
    Explainable
    Every rule can be inspected later in Results.
    )} - - {/* Summary */} -
    -

    Configuration Summary

    -
    -
    -

    Task

    -

    {taskType || "Not set"}

    -
    -
    -

    Target

    -

    {targetColumn || "Not set"}

    -
    -
    -

    Features

    -

    {selectedFeatures.length}

    -
    -
    -

    Models

    -

    {selectedModels.length}

    -
    + +
    +
    Configuration summary
    +
    +
    Task
    {taskType}
    +
    Target
    {targetColumn || "Not set"}
    +
    Features
    {selectedFeatures.length}
    +
    Models
    {selectedModels.length}
    +
    Optimization
    {optimizationEnabled ? "On" : "Off"}
    - -
    - {onBack && ( - - )} -
    - : } +
    + -
    diff --git a/Frontend/src/components/playground/PredictionStep.tsx b/Frontend/src/components/playground/PredictionStep.tsx index d17682e..d819774 100644 --- a/Frontend/src/components/playground/PredictionStep.tsx +++ b/Frontend/src/components/playground/PredictionStep.tsx @@ -1,148 +1,165 @@ import { useState } from "react"; -import { Sparkles, Upload, Download, AlertCircle, Loader2 } from "lucide-react"; +import { + AlertCircle, + CheckCircle2, + Download, + FileSpreadsheet, + History, + Loader2, + Sparkles, + Upload, +} from "lucide-react"; + +import { Alert, AlertDescription } from "@/components/ui/alert"; +import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; -import { Alert, AlertDescription } from "@/components/ui/alert"; -import { useToast } from "@/hooks/use-toast"; import { useExperiment } from "@/contexts/ExperimentContext"; +import { useToast } from "@/hooks/use-toast"; import { predictionAPI } from "@/services/apiService"; +interface SinglePredictionResult { + prediction: string; + confidence?: number | null; + probabilities?: Record | null; +} + +interface BatchPredictionResult { + prediction_id: string; + total_predictions: number; + download_url: string; +} + +interface PredictionHistoryItem { + id: string; + total_predictions: number; + created_at: string; +} + +const MAX_BATCH_MB = 100; + +const messageFromError = (error: any, fallback: string) => + error?.response?.data?.detail || error?.message || fallback; + const PredictionStep = () => { const { currentExperiment } = useExperiment(); const { toast } = useToast(); - - const [loading, setLoading] = useState(false); - const [predictionResult, setPredictionResult] = useState(null); - const [batchResult, setBatchResult] = useState(null); - const [inputValues, setInputValues] = useState>({}); - const [error, setError] = useState(null); - const [predictionHistory, setPredictionHistory] = useState([]); + + const [inputValues, setInputValues] = useState>({}); + const [predictionResult, setPredictionResult] = useState(null); + const [batchResult, setBatchResult] = useState(null); + const [predictionHistory, setPredictionHistory] = useState([]); const [showHistory, setShowHistory] = useState(false); - - // Get feature names from experiment config + const [singleLoading, setSingleLoading] = useState(false); + const [batchLoading, setBatchLoading] = useState(false); + const [historyLoading, setHistoryLoading] = useState(false); + const [error, setError] = useState(null); + const selectedFeatures = currentExperiment?.config?.selectedFeatures || []; - + const numericFeatures = new Set(currentExperiment?.config?.featureTypes?.numerical || []); + const hasCompletedResult = Boolean(currentExperiment?.results) || currentExperiment?.status === "completed"; + const handleSinglePredict = async () => { - if (!currentExperiment) { - setError("No experiment selected"); + if (!currentExperiment) return; + + const missingFeatures = selectedFeatures.filter((feature) => { + const value = inputValues[feature]; + return value === undefined || value.trim() === ""; + }); + if (missingFeatures.length) { + setError(`Fill in all required features: ${missingFeatures.join(", ")}`); return; } - - // Validate all features are filled - const missingFeatures = selectedFeatures.filter(feature => !inputValues[feature]); - if (missingFeatures.length > 0) { - setError(`Please fill in all features: ${missingFeatures.join(", ")}`); + + const features = Object.fromEntries( + selectedFeatures.map((feature) => { + const raw = inputValues[feature]; + return [feature, numericFeatures.has(feature) ? Number(raw) : raw]; + }), + ); + + if (selectedFeatures.some((feature) => numericFeatures.has(feature) && Number.isNaN(features[feature]))) { + setError("One or more numerical features contain an invalid number."); return; } - - setLoading(true); + + setSingleLoading(true); setError(null); - try { - // Convert values to numbers where appropriate - const features = Object.fromEntries( - Object.entries(inputValues).map(([key, value]) => [key, isNaN(Number(value)) ? value : Number(value)]) - ); - - const result = await predictionAPI.single(currentExperiment.id, features); + const result = (await predictionAPI.single(currentExperiment.id, features)) as SinglePredictionResult; setPredictionResult(result); - - toast({ - title: "Prediction Complete!", - description: `Result: ${result.prediction}` - }); - } catch (error: any) { - const errorMsg = error.response?.data?.detail || error.message || "Unknown error"; - setError(errorMsg); - toast({ - title: "Prediction Failed", - description: errorMsg, - variant: "destructive" - }); + toast({ title: "Prediction complete", description: `Predicted value: ${result.prediction}` }); + } catch (predictionError: any) { + const message = messageFromError(predictionError, "Prediction failed."); + setError(message); + toast({ title: "Prediction failed", description: message, variant: "destructive" }); } finally { - setLoading(false); + setSingleLoading(false); } }; - + const handleBatchPredict = async (file: File) => { - if (!currentExperiment) { - setError("No experiment selected"); + if (!currentExperiment) return; + if (!file.name.toLowerCase().endsWith(".csv")) { + setError("Batch prediction currently accepts CSV files only."); return; } - - setLoading(true); + if (file.size > MAX_BATCH_MB * 1024 * 1024) { + setError(`Batch CSV files must be ${MAX_BATCH_MB} MB or smaller.`); + return; + } + + setBatchLoading(true); setError(null); - try { - const result = await predictionAPI.batch(currentExperiment.id, file); + const result = (await predictionAPI.batch(currentExperiment.id, file)) as BatchPredictionResult; setBatchResult(result); - - toast({ - title: "Batch Prediction Complete!", - description: `${result.total_predictions} predictions made` - }); - } catch (error: any) { - const errorMsg = error.response?.data?.detail || error.message || "Unknown error"; - setError(errorMsg); - toast({ - title: "Batch Prediction Failed", - description: errorMsg, - variant: "destructive" - }); + toast({ title: "Batch prediction complete", description: `${result.total_predictions.toLocaleString()} rows predicted.` }); + } catch (predictionError: any) { + const message = messageFromError(predictionError, "Batch prediction failed."); + setError(message); + toast({ title: "Batch prediction failed", description: message, variant: "destructive" }); } finally { - setLoading(false); - } - }; - - const handleFileChange = (e: React.ChangeEvent) => { - const file = e.target.files?.[0]; - if (file) { - if (!file.name.endsWith('.csv')) { - setError("Please upload a CSV file"); - return; - } - handleBatchPredict(file); + setBatchLoading(false); } }; - + const handleDownload = async (predictionId?: string) => { const id = predictionId || batchResult?.prediction_id; - if (id) { - try { - await predictionAPI.download(id); - } catch (err) { - console.error('Download failed:', err); - setError('Failed to download predictions. Please try again.'); - } + if (!id) return; + try { + await predictionAPI.download(id); + } catch (downloadError: any) { + const message = messageFromError(downloadError, "Prediction download failed."); + setError(message); } }; - + const loadPredictionHistory = async () => { if (!currentExperiment) return; - + setHistoryLoading(true); try { const result = await predictionAPI.getHistory(currentExperiment.id); setPredictionHistory(result.predictions || []); setShowHistory(true); - } catch (err) { - console.error('Failed to load history:', err); + } catch (historyError: any) { + setError(messageFromError(historyError, "Could not load prediction history.")); + } finally { + setHistoryLoading(false); } }; - + if (!currentExperiment) { return ( -
    - - - - No experiment selected. Please create an experiment first. - - -
    + + + No experiment is selected. + ); } - + return (
    {error && ( @@ -151,244 +168,206 @@ const PredictionStep = () => { {error} )} - -
    -

    - - Make Predictions -

    - -
    - {/* Single Prediction */} -
    -

    Single Prediction

    -

    - Enter feature values to get a prediction from your trained model -

    - -
    - {selectedFeatures.length === 0 ? ( - - - - No features configured. Please configure your experiment first. - - - ) : ( - selectedFeatures.map((feature: string) => ( -
    - - setInputValues({...inputValues, [feature]: e.target.value})} - disabled={loading} - /> -
    - )) - )} + + + +
    +
    +
    + Prediction workspace +
    + Use your best trained model + Run one prediction interactively or score an entire CSV using the same fitted preprocessing pipeline as training.
    - - + {selectedFeatures.length} required features
    - - {/* Batch Prediction */} -
    -

    Batch Prediction

    -

    - Upload a CSV file with the same features as your training data -

    - -
    - -
    -

    Upload CSV for batch predictions

    -

    - File must contain columns: {selectedFeatures.join(", ")} -

    -
    - -
    - - + + + + {!hasCompletedResult && ( + + + + Predictions require at least one successful completed training run. If prediction fails, return to Train and Results first. + + + )} + +
    + + + Single prediction + Enter one observation. Numerical features accept zero and decimal values normally. + + + {!selectedFeatures.length ? ( + + + Configure the experiment features before making predictions. + + ) : ( +
    + {selectedFeatures.map((feature: string) => { + const numeric = numericFeatures.has(feature); + return ( +
    +
    + + {numeric ? "number" : "text/category"} +
    + { + setInputValues((current) => ({ ...current, [feature]: event.target.value })); + if (error) setError(null); + }} + disabled={singleLoading} + /> +
    + ); + })}
    -
    - + )} + + + + + + + + Batch prediction + Upload a CSV containing all required feature columns. Extra columns are preserved in the exported result. + + + + {batchResult && ( -
    -

    - โœ“ Batch Prediction Complete! -

    -

    - {batchResult.total_predictions} predictions generated successfully -

    +
    +
    +
    +
    Batch complete
    +
    {batchResult.total_predictions.toLocaleString()} predictions generated.
    +
    + +
    )} -
    -
    + +
    - - {/* Single Prediction Result */} + {predictionResult && ( -
    -

    Prediction Result

    - -
    - {/* Prediction Value */} -
    -

    Prediction

    -

    {predictionResult.prediction}

    -
    - - {/* Confidence Score */} - {predictionResult.confidence !== null && predictionResult.confidence !== undefined && ( -
    -

    Confidence

    -

    - {(predictionResult.confidence * 100).toFixed(1)}% -

    -
    -
    + + + Prediction result + + +
    +
    +
    Prediction
    +
    {predictionResult.prediction}
    +
    + {predictionResult.confidence != null && ( +
    +
    Confidence
    +
    {(predictionResult.confidence * 100).toFixed(1)}%
    +
    + )} +
    +
    Input features
    +
    {selectedFeatures.length}
    +
    Processed by the saved V3 pipeline
    - )} - - {/* Class Probabilities */} - {predictionResult.probabilities && ( -
    -

    Class Probabilities

    -
    - {Object.entries(predictionResult.probabilities).map(([key, value]) => ( -
    - {key}: -
    -
    -
    -
    - - {((value as number) * 100).toFixed(1)}% - +
    + + {predictionResult.probabilities && Object.keys(predictionResult.probabilities).length > 0 && ( +
    +
    Class probabilities
    +
    + {Object.entries(predictionResult.probabilities) + .sort((a, b) => b[1] - a[1]) + .map(([label, probability]) => ( +
    + {label} +
    + {(probability * 100).toFixed(1)}%
    -
    - ))} + ))}
    )} -
    - - {/* Input Summary */} -
    -

    Input Features:

    -
    - {Object.entries(inputValues).map(([key, value]) => ( -
    - {key}: - {value} -
    - ))} + + + )} + + + +
    +
    + Batch history + Authenticated exports generated for this experiment.
    +
    -
    - )} - - {/* Prediction History */} -
    -
    -

    - - Prediction History -

    - -
    - + {showHistory && ( -
    - {predictionHistory.length === 0 ? ( -

    - No prediction history yet. Make batch predictions to see them here. -

    + + {!predictionHistory.length ? ( +
    + No batch predictions yet. +
    ) : ( - predictionHistory.map((pred) => ( -
    -
    -

    - {pred.total_predictions} predictions -

    -

    - {new Date(pred.created_at).toLocaleString()} -

    +
    + {predictionHistory.map((item) => ( +
    +
    +
    {item.total_predictions.toLocaleString()} predictions
    +
    {new Date(item.created_at).toLocaleString()}
    +
    +
    - -
    - )) + ))} +
    )} -
    +     )} -
    +
    ); }; diff --git a/Frontend/src/components/playground/ResultsStep.tsx b/Frontend/src/components/playground/ResultsStep.tsx index 5fcc8b0..b635257 100644 --- a/Frontend/src/components/playground/ResultsStep.tsx +++ b/Frontend/src/components/playground/ResultsStep.tsx @@ -1,804 +1,304 @@ -import React, { useEffect, useState } from 'react'; -import { Card, CardContent, CardDescription, CardHeader, CardTitle } from '@/components/ui/card'; -import { Button } from '@/components/ui/button'; -import { Badge } from '@/components/ui/badge'; -import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from '@/components/ui/table'; -import { Eye, Download, Loader2, ArrowLeft, AlertTriangle, CheckCircle2, AlertCircle, Sparkles, Info } from 'lucide-react'; -import { BarChart, Bar, XAxis, YAxis, CartesianGrid, Tooltip, Legend, ResponsiveContainer, Cell } from 'recharts'; -import apiService from '@/services/apiService'; +import { useCallback, useEffect, useState } from "react"; +import { + AlertCircle, + ArrowLeft, + ChevronLeft, + ChevronRight, + Download, + Eye, + Loader2, + Sparkles, +} from "lucide-react"; + +import RunModelAnalysis, { type ModelResult } from "@/components/playground/RunModelAnalysis"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent } from "@/components/ui/card"; +import apiService from "@/services/apiService"; interface ResultsStepProps { experimentId: string; onBack: () => void; } -interface RunListItem { - id: string; - run_number: number; - status: string; - started_at?: string; - completed_at?: string; - duration_seconds?: number; - results_summary?: any; - created_at: string; -} - -interface ConfusionMatrix { - matrix: number[][]; - labels: string[]; +interface BestModel { + model_type: string; + display_name: string; + metric: string; + value: number; } -interface FeatureImportance { - features: string[]; - importance: number[]; +interface ResultsSummary { + total_models?: number; + successful?: number; + failed?: number; + best_model?: BestModel; } -interface Metrics { - // New structure (nested train/test) - train?: { - accuracy?: number; - precision?: number; - recall?: number; - f1_score?: number; - r2_score?: number; - mae?: number; - rmse?: number; - mse?: number; - }; - test?: { - accuracy?: number; - precision?: number; - recall?: number; - f1_score?: number; - r2_score?: number; - mae?: number; - rmse?: number; - mse?: number; - }; - // Old structure (flat) - for backward compatibility - accuracy?: number; - precision?: number; - recall?: number; - f1_score?: number; - r2_score?: number; - mae?: number; - rmse?: number; - mse?: number; +interface RunListItem { + id: string; + run_number: number; + status: "pending" | "running" | "completed" | "failed" | "cancelled"; + started_at?: string | null; + completed_at?: string | null; + duration_seconds?: number | null; + results_summary?: ResultsSummary | null; + error_message?: string | null; + created_at: string; } interface RunDetails { id: string; run_number: number; - status: string; - config_snapshot: any; + status: RunListItem["status"]; + config_snapshot?: Record; results: { - task_type?: string; - dataset_info?: { - total_samples: number; - train_samples: number; - test_samples: number; - n_features: number; - feature_names: string[]; - }; - models: Array<{ - model_type: string; - display_name: string; - metrics: Metrics; - feature_importance?: FeatureImportance; - confusion_matrix?: ConfusionMatrix; - error?: string; - }>; - best_model?: { - model_type: string; - display_name: string; - metric: string; - value: number; - }; - summary: { - total_models: number; - successful: number; - failed: number; - }; + task_type?: "classification" | "regression"; + models?: ModelResult[]; + best_model?: BestModel; + summary?: ResultsSummary; + training_config?: Record; + dataset_info?: Record | null; }; - artifacts?: any; + error_message?: string | null; + started_at?: string | null; + completed_at?: string | null; + duration_seconds?: number | null; created_at: string; } -const ResultsStep: React.FC = ({ experimentId, onBack }) => { +interface RunsResponse { + runs?: RunListItem[]; + total_pages?: number; +} + +const messageFromError = (error: unknown, fallback: string) => + error instanceof Error && error.message ? error.message : fallback; + +const formatDuration = (seconds?: number | null) => { + if (seconds == null) return "โ€”"; + if (seconds < 60) return `${seconds}s`; + return `${Math.floor(seconds / 60)}m ${Math.round(seconds % 60)}s`; +}; + +const statusBadge = (status: RunListItem["status"]) => { + if (status === "completed") return Completed; + if (status === "running") return Running; + if (status === "failed") return Failed; + if (status === "cancelled") return Cancelled; + return Pending; +}; + +const ResultsStep = ({ experimentId, onBack }: ResultsStepProps) => { const [runs, setRuns] = useState([]); const [selectedRun, setSelectedRun] = useState(null); const [loading, setLoading] = useState(true); - const [detailsLoading, setDetailsLoading] = useState(false); + const [detailsLoading, setDetailsLoading] = useState(null); + const [error, setError] = useState(null); const [page, setPage] = useState(1); const [totalPages, setTotalPages] = useState(1); - const [fetchError, setFetchError] = useState(null); - useEffect(() => { - if (!experimentId) { - setFetchError('No experiment ID provided'); - setLoading(false); - return; - } - fetchRuns(page); - }, [experimentId, page]); - - const fetchRuns = async (pageNum: number) => { - if (!experimentId) { - setFetchError('No experiment ID provided'); - setLoading(false); - return; - } - + const fetchRuns = useCallback(async () => { try { setLoading(true); - setFetchError(null); - const response = await apiService.training.listRuns(experimentId, pageNum); - - if (!response || typeof response !== 'object') { - throw new Error('Invalid response format'); - } - - setRuns(response.runs || []); - setTotalPages(response.total_pages || 1); - } catch (error: any) { - console.error('Failed to fetch runs:', error); - const errorMessage = error.response?.data?.detail || error.message || 'Failed to load training runs'; - setFetchError(errorMessage); + setError(null); + const response = (await apiService.training.listRuns(experimentId, page, 10)) as RunsResponse; + setRuns(response.runs ?? []); + setTotalPages(Math.max(1, response.total_pages ?? 1)); + } catch (fetchError: unknown) { + setError(messageFromError(fetchError, "Failed to load training runs.")); setRuns([]); } finally { setLoading(false); } - }; + }, [experimentId, page]); - const fetchRunDetails = async (runId: string) => { + useEffect(() => { + void fetchRuns(); + }, [fetchRuns]); + + const openRun = async (runId: string) => { try { - setDetailsLoading(true); - setFetchError(null); - const details = await apiService.training.getRunDetails(runId); - - if (!details || typeof details !== 'object') { - throw new Error('Invalid response format'); - } - - // Validate required fields - if (!details.results || !details.config_snapshot) { - throw new Error('Incomplete run data'); - } - - setSelectedRun(details); - } catch (error: any) { - console.error('Failed to fetch run details:', error); - const errorMessage = error.response?.data?.detail || error.message || 'Failed to load run details'; - setFetchError(errorMessage); + setDetailsLoading(runId); + setError(null); + const detail = (await apiService.training.getRunDetails(runId)) as RunDetails; + if (!detail?.results) throw new Error("This run does not contain results yet."); + setSelectedRun(detail); + } catch (detailError: unknown) { + setError(messageFromError(detailError, "Failed to load run details.")); } finally { - setDetailsLoading(false); + setDetailsLoading(null); } }; - const formatDuration = (seconds?: number) => { - if (!seconds) return 'N/A'; - const mins = Math.floor(seconds / 60); - const secs = seconds % 60; - return `${mins}m ${secs}s`; - }; + const downloadRunReport = (run: RunDetails) => { + const models = run.results.models ?? []; + const sanitizedModels = models.map((model) => ({ + model_type: model.model_type, + display_name: model.display_name, + metrics: model.metrics, + feature_importance: model.feature_importance, + confusion_matrix: model.confusion_matrix, + hyperparameters: model.hyperparameters, + hyperparameter_tuning: model.hyperparameter_tuning, + error: model.error, + })); - const getStatusBadge = (status: string) => { - switch (status) { - case 'completed': - return Completed; - case 'running': - return Running; - case 'failed': - return Failed; - default: - return Pending; - } - }; + const report = { + report_format: "nocodeml-run-report-v3", + exported_at: new Date().toISOString(), + experiment_id: experimentId, + run: { + id: run.id, + run_number: run.run_number, + status: run.status, + created_at: run.created_at, + started_at: run.started_at, + completed_at: run.completed_at, + duration_seconds: run.duration_seconds, + error_message: run.error_message, + config_snapshot: run.config_snapshot ?? {}, + results: { + task_type: run.results.task_type, + best_model: run.results.best_model, + summary: run.results.summary, + training_config: run.results.training_config, + dataset_info: run.results.dataset_info, + models: sanitizedModels, + }, + }, + }; - const getOverfittingIndicator = (trainScore: number, testScore: number, isClassification: boolean) => { - const gap = Math.abs(trainScore - testScore); - const gapPercent = (gap * 100).toFixed(1); - - if (gap < 0.05) { - return ( -
    - - Good generalization (gap: {gapPercent}%) -
    - ); - } else if (gap < 0.15) { - return ( -
    - - Slight overfitting (gap: {gapPercent}%) -
    - ); - } else { - return ( -
    - - High overfitting (gap: {gapPercent}%) -
    - ); - } + const blob = new Blob([JSON.stringify(report, null, 2)], { type: "application/json" }); + const url = URL.createObjectURL(blob); + const anchor = document.createElement("a"); + anchor.href = url; + anchor.download = `nocodeml-run-${run.run_number}-report.json`; + document.body.appendChild(anchor); + anchor.click(); + anchor.remove(); + URL.revokeObjectURL(url); }; if (selectedRun) { - // Detail View - Determine task type for dynamic metrics - // Priority: results.task_type (what was actually used) > config_snapshot.taskType (what was configured) - const taskType = selectedRun.results?.task_type || selectedRun.config_snapshot?.taskType || 'classification'; - const isClassification = taskType === 'classification'; - - // Safety check for results - if (!selectedRun.results || !selectedRun.results.models) { - return ( -
    -
    -

    Run #{selectedRun.run_number}

    - -
    - - -

    No results available for this run

    -     -     -
    - ); - } - + const taskType = selectedRun.results.task_type ?? ((selectedRun.config_snapshot?.taskType as "classification" | "regression" | undefined) ?? "classification"); + const models = selectedRun.results.models ?? []; + const summary = selectedRun.results.summary; + return ( -
    -
    +
    +
    -

    Run #{selectedRun.run_number}

    -

    - {new Date(selectedRun.created_at).toLocaleString()} -

    -
    -
    - - {getStatusBadge(selectedRun.status)} -
    -
    - - {/* Results Summary */} - - - Summary - - -
    -
    -

    Total Models

    -

    {selectedRun.results.summary?.total_models || 0}

    -
    -
    -

    Successful

    -

    {selectedRun.results.summary?.successful || 0}

    -
    -
    -

    Failed

    -

    {selectedRun.results.summary?.failed || 0}

    -
    +
    +

    Run #{selectedRun.run_number}

    + {statusBadge(selectedRun.status)}
    - +

    + {new Date(selectedRun.created_at).toLocaleString()} ยท {formatDuration(selectedRun.duration_seconds)} +

    +
    +
    {selectedRun.results.best_model && ( -
    -

    Best Model

    -

    {selectedRun.results.best_model.display_name || 'Unknown'}

    -

    - {selectedRun.results.best_model.metric || 'Score'}: {selectedRun.results.best_model.value?.toFixed(4) || 'N/A'} -

    +
    +
    Best candidate
    +
    {selectedRun.results.best_model.display_name}
    +
    + {selectedRun.results.best_model.metric}: {selectedRun.results.best_model.value.toFixed(4)} +
    )} - - - - {/* Model Results */} - - - Model Performance - - - - - - Model - {isClassification ? ( - <> - Accuracy - Precision - Recall - F1 Score - - ) : ( - <> - Rยฒ Score - MAE - RMSE - MSE - - )} - Status - - - - {selectedRun.results.models && selectedRun.results.models.length > 0 ? ( - selectedRun.results.models.map((model) => ( - - {model.display_name || model.model_type} - {model.error ? ( - <> - {model.error} - Failed - - ) : model.metrics ? ( - // Support both new structure (metrics.test) and old structure (metrics.accuracy) - isClassification ? ( - <> - {(model.metrics.test?.accuracy ?? model.metrics.accuracy)?.toFixed(4) || 'N/A'} - {(model.metrics.test?.precision ?? model.metrics.precision)?.toFixed(4) || 'N/A'} - {(model.metrics.test?.recall ?? model.metrics.recall)?.toFixed(4) || 'N/A'} - {(model.metrics.test?.f1_score ?? model.metrics.f1_score)?.toFixed(4) || 'N/A'} - Success - - ) : ( - <> - {(model.metrics.test?.r2_score ?? model.metrics.r2_score)?.toFixed(4) || 'N/A'} - {(model.metrics.test?.mae ?? model.metrics.mae)?.toFixed(4) || 'N/A'} - {(model.metrics.test?.rmse ?? model.metrics.rmse)?.toFixed(4) || 'N/A'} - {(model.metrics.test?.mse ?? model.metrics.mse)?.toFixed(4) || 'N/A'} - Success - - ) - ) : ( - <> - No metrics available - Unknown - - )} - - )) - ) : ( - - - No model results available - - - )} - -
    -     -     - - {/* Model Analysis Section - Loop through each successful model */} - {selectedRun.results.models && selectedRun.results.models.filter(m => m.metrics && !m.error).map((model) => { - // Extract train and test scores with null safety (support old and new structure) - const trainScore = isClassification - ? (model.metrics?.train?.accuracy ?? 0) - : (model.metrics?.train?.r2_score ?? 0); - const testScore = isClassification - ? (model.metrics?.test?.accuracy ?? model.metrics?.accuracy ?? 0) - : (model.metrics?.test?.r2_score ?? model.metrics?.r2_score ?? 0); - - return ( - - - - {model.display_name} - Detailed Analysis - {model.model_type === selectedRun.results.best_model?.model_type && ( - Best Model - )} - - - - {/* Hyperparameter Optimization Results - Transparent Expert System */} - {model.hyperparameter_tuning?.enabled && ( -
    - {/* Expert System Header */} -
    -
    - -
    - Expert System โ€” Optimization Summary -
    -
    -

    - Our Expert System adjusted your model based on your data's specific characteristics - {model.hyperparameter_tuning.dataset_info && ( - - {' '}(Size: {model.hyperparameter_tuning.dataset_info.n_samples?.toLocaleString()} samples, Features: {model.hyperparameter_tuning.dataset_info.n_features}) - - )}. -

    -
    - - {/* Summary Metrics */} -
    -
    -
    Engine
    -
    - {model.hyperparameter_tuning.engine || 'NoCodeML Rules Engine'} -
    -
    - {model.hyperparameter_tuning.method} -
    -
    - -
    -
    Rules Evaluated
    -
    - {model.hyperparameter_tuning.rules_evaluated ?? model.hyperparameter_tuning.applied_rules?.length ?? 0} -
    -
    - {model.hyperparameter_tuning.cv_strategy} -
    -
    - -
    -
    Final Configuration
    -
    - {model.hyperparameter_tuning.test_score?.toFixed(4)} -
    -
    - Test Score -
    -
    - -
    -
    Rules Applied
    -
    - {model.hyperparameter_tuning.rules_applied ?? model.hyperparameter_tuning.applied_rules?.filter((r: any) => r.parameter !== 'โ€”').length ?? 0} -
    -
    - Parameter adjustments -
    -
    -
    - - {/* Reasoning Table โ€” Step-by-Step Logic */} - {model.hyperparameter_tuning.applied_rules && model.hyperparameter_tuning.applied_rules.length > 0 && ( -
    -
    -
    - - Step-by-Step Reasoning โ€” Why Each Parameter Was Changed -
    -
    - - - - Parameter - Before - After - The "Why" (Reasoning) - - - - {model.hyperparameter_tuning.applied_rules.map((rule: any, idx: number) => ( - - - {rule.parameter} - - - {typeof rule.original_value === 'object' ? JSON.stringify(rule.original_value) : String(rule.original_value)} - - - {typeof rule.value === 'object' ? JSON.stringify(rule.value) : String(rule.value)} - - - {rule.reason} - - - ))} - -
    -
    - )} - - {/* Optimized Parameters Details */} -
    - - - โ–ถ - View Final Hyperparameters - - -
    - 
    -                        {JSON.stringify(model.hyperparameter_tuning.best_params, null, 2)}
-
    -
    -
    -
    - )} + +
    +
    - {/* Train vs Test Comparison */} -
    -

    - Overfitting Check - Generalization -

    -
    -
    -

    Train Score

    -

    - {trainScore > 0 ? trainScore.toFixed(4) : 'N/A'} -

    -
    -
    -

    Test Score

    -

    - {testScore > 0 ? testScore.toFixed(4) : 'N/A'} -

    -
    -
    - {trainScore > 0 && testScore > 0 && getOverfittingIndicator(trainScore, testScore, isClassification)} -
    +
    +
    Models
    {summary?.total_models ?? models.length}
    +
    Successful
    {summary?.successful ?? models.filter((model) => !model.error).length}
    +
    Failed
    {summary?.failed ?? models.filter((model) => model.error).length}
    +
    -
    - {/* Show message if no visualizations available */} - {!model.confusion_matrix && !model.feature_importance && ( -
    - -

    Advanced Visualizations Not Available

    -

    - This training run was completed before visualization features were added. -

    -
    - - Start a new training run to see confusion matrix & feature importance -
    -
    - )} - - {/* Confusion Matrix */} - {isClassification && model.confusion_matrix && ( -
    -

    - Confusion Matrix - Predictions -

    -
    -
    - - - - - {model.confusion_matrix.labels.map((label) => ( - - ))} - - - - {model.confusion_matrix.matrix.map((row, i) => ( - - - {row.map((val, j) => { - const total = row.reduce((a, b) => a + b, 0); - const percentage = total > 0 ? ((val / total) * 100).toFixed(0) : '0'; - const isCorrect = i === j; - return ( - - ); - })} - - ))} - -
    -
    Predicted
    -
    {label}
    -
    -
    Actual
    -
    {model.confusion_matrix!.labels[i]}
    -     		0 - ? 'bg-red-500/20 hover:bg-red-500/30' - : 'bg-muted/30 hover:bg-muted/50' - }`} - > -
    {val}
    -
    ({percentage}%)
    -
    -
    -
    -
    -
    - Correct predictions -
    -
    -
    - Incorrect predictions -
    -
    -
    -
    - )} + {selectedRun.error_message && ( +
    {selectedRun.error_message}
    + )} - {/* Feature Importance */} - {model.feature_importance && - model.feature_importance.features && - model.feature_importance.importance && - model.feature_importance.features.length > 0 && ( -
    -

    - Feature Importance - Top 10 -

    -
    - - ({ - feature: feature && feature.length > 20 ? feature.substring(0, 17) + '...' : feature || `Feature ${idx}`, - importance: model.feature_importance!.importance[idx] || 0 - }))} - layout="vertical" - margin={{ top: 5, right: 30, left: 100, bottom: 5 }} - > - - - - - - {model.feature_importance!.features.slice(0, 10).map((entry, index) => ( - - ))} - - - -
    -
    -
    - Top 3 features -
    -
    -
    - Other features -
    -
    -
    -
    - )} -
    -     -     - ); - })} + {models.length ? ( + models.map((model) => ( + + )) + ) : ( + No model results are available for this run. + )}
    ); } - // List View return ( -
    -
    +
    +
    -

    Training Runs

    -

    - View and compare all training runs for this experiment -

    +

    Training runs

    +

    Compare runs without losing the configuration that produced them. Open any run to export its reproducibility report.

    - +
    - {fetchError && ( - - -

    {fetchError}

    - -     -     + {error && ( +
    + +
    {error}
    + +
    )} {loading ? ( -
    - -
    - ) : runs.length === 0 && !fetchError ? ( - +
    + ) : runs.length === 0 ? ( + -

    No training runs yet. Start training to see results here.

    + +
    No training runs yet
    +

    Complete model configuration and start a run to see comparisons here.

         - ) : !fetchError && runs.length > 0 ? ( - - - All Runs ({runs.length}) - - - - - - Run # - Date - Status - Duration - Best Model - Actions - - - - {runs.map((run) => ( - - #{run.run_number} - {new Date(run.created_at).toLocaleDateString()} - {getStatusBadge(run.status)} - {formatDuration(run.duration_seconds)} - - {run.results_summary?.best_model ? ( -
    -

    {run.results_summary.best_model.display_name}

    -

    - {run.results_summary.best_model.value.toFixed(3)} -

    -
    - ) : ( - 'N/A' - )} -     - - - -     - ))} -     -
    - - {/* Pagination */} - {totalPages > 1 && ( -
    - - - Page {page} of {totalPages} - - -
    - )} -     -     - ) : null} + +     + ))} + + {totalPages > 1 && ( +
    + + Page {page} of {totalPages} + +
    + )} +
    + )}
    ); }; diff --git a/Frontend/src/components/playground/RunModelAnalysis.tsx b/Frontend/src/components/playground/RunModelAnalysis.tsx new file mode 100644 index 0000000..67bc0f4 --- /dev/null +++ b/Frontend/src/components/playground/RunModelAnalysis.tsx @@ -0,0 +1,253 @@ +import { useMemo } from "react"; +import { CheckCircle2, Info, Sparkles, Trophy } from "lucide-react"; +import { + Bar, + BarChart, + CartesianGrid, + ResponsiveContainer, + Tooltip, + XAxis, + YAxis, +} from "recharts"; + +import { Badge } from "@/components/ui/badge"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; + +export interface MetricGroup { + accuracy?: number; + precision?: number; + recall?: number; + f1_score?: number; + roc_auc?: number; + r2_score?: number; + mae?: number; + rmse?: number; + mse?: number; +} + +export interface Metrics extends MetricGroup { + train?: MetricGroup; + test?: MetricGroup; +} + +export interface ModelResult { + model_type: string; + display_name: string; + metrics?: Metrics; + feature_importance?: { features: string[]; importance: number[] }; + confusion_matrix?: { matrix: number[][]; labels: string[] }; + hyperparameters?: Record; + hyperparameter_tuning?: { + enabled?: boolean; + engine?: string; + method?: string; + rules_evaluated?: number; + rules_applied?: number; + cv_strategy?: string; + test_score?: number; + best_params?: Record; + applied_rules?: Array<{ + parameter: string; + original_value: unknown; + value: unknown; + reason: string; + }>; + dataset_info?: { n_samples?: number; n_features?: number }; + }; + error?: string; +} + +const metricValue = (metrics: Metrics | undefined, task: "classification" | "regression") => { + if (!metrics) return null; + if (task === "classification") return metrics.test?.accuracy ?? metrics.accuracy ?? null; + return metrics.test?.r2_score ?? metrics.r2_score ?? null; +}; + +const RunModelAnalysis = ({ + model, + taskType, + isBest, +}: { + model: ModelResult; + taskType: "classification" | "regression"; + isBest: boolean; +}) => { + const test = model.metrics?.test ?? model.metrics; + const train = model.metrics?.train; + const score = metricValue(model.metrics, taskType); + + const featureData = useMemo( + () => + (model.feature_importance?.features ?? []).slice(0, 10).map((feature, index) => ({ + feature: feature.length > 26 ? `${feature.slice(0, 23)}โ€ฆ` : feature, + importance: model.feature_importance?.importance[index] ?? 0, + })), + [model.feature_importance], + ); + + return ( + + +
    +
    +
    + {model.display_name || model.model_type} + {isBest && ( + + Best model + + )} +
    +

    {model.model_type}

    +
    + {model.error ? ( + Training failed + ) : score != null ? ( +
    +
    {score.toFixed(4)}
    +
    {taskType === "classification" ? "test accuracy" : "test Rยฒ"}
    +
    + ) : null} +
    +     + + + {model.error ? ( +
    {model.error}
    + ) : ( + <> +
    + {(taskType === "classification" + ? [ + ["Accuracy", test?.accuracy ?? model.metrics?.accuracy], + ["Precision", test?.precision ?? model.metrics?.precision], + ["Recall", test?.recall ?? model.metrics?.recall], + ["F1 score", test?.f1_score ?? model.metrics?.f1_score], + ] + : [ + ["Rยฒ", test?.r2_score ?? model.metrics?.r2_score], + ["MAE", test?.mae ?? model.metrics?.mae], + ["RMSE", test?.rmse ?? model.metrics?.rmse], + ["MSE", test?.mse ?? model.metrics?.mse], + ] + ).map(([label, value]) => ( +
    +
    {label}
    +
    {typeof value === "number" ? value.toFixed(4) : "โ€”"}
    +
    + ))} +
    + + {train && score != null && ( +
    +
    + Generalization check +
    +
    +
    + Train score + {(taskType === "classification" ? train.accuracy : train.r2_score)?.toFixed(4) ?? "โ€”"} +
    +
    + Test score {score.toFixed(4)} +
    +
    +
    + )} + + {model.hyperparameter_tuning?.enabled && ( +
    +
    +
    + +
    +
    +
    Expert optimization
    +
    + {model.hyperparameter_tuning.engine || "NoCodeML rules engine"} ยท {model.hyperparameter_tuning.cv_strategy || model.hyperparameter_tuning.method || "configured strategy"} +
    +
    +
    + +
    +
    Rules evaluated
    {model.hyperparameter_tuning.rules_evaluated ?? model.hyperparameter_tuning.applied_rules?.length ?? 0}
    +
    Rules applied
    {model.hyperparameter_tuning.rules_applied ?? 0}
    +
    Samples
    {model.hyperparameter_tuning.dataset_info?.n_samples?.toLocaleString() ?? "โ€”"}
    +
    Features
    {model.hyperparameter_tuning.dataset_info?.n_features ?? "โ€”"}
    +
    + + {model.hyperparameter_tuning.applied_rules?.length ? ( +
    + + ParameterBeforeAfterReason + + {model.hyperparameter_tuning.applied_rules.map((rule, index) => ( + + {rule.parameter} + {String(rule.original_value)} + {String(rule.value)} + {rule.reason} + + ))} + +
    +
    + ) : null} + + {(model.hyperparameter_tuning.best_params || model.hyperparameters) && ( +
    + Final hyperparameters + 
    {JSON.stringify(model.hyperparameter_tuning.best_params || model.hyperparameters, null, 2)}
    +
    + )} +
    + )} + +
    + {taskType === "classification" && model.confusion_matrix && ( +
    +
    Confusion matrix
    +
    + + )} + + {model.confusion_matrix.matrix.map((row, rowIndex) => ( + + + {row.map((value, columnIndex) => ( + + ))} + + ))} + +
    {model.confusion_matrix.labels.map((label) => Pred {label}
    Actual {model.confusion_matrix?.labels[rowIndex] ?? rowIndex}{value}
    +
    +
    + )} + + {featureData.length > 0 && ( +
    +
    Feature importance
    +
    + + + + + + + + + +
    +
    + )} +
    + + )} +     +     + ); +}; + +export default RunModelAnalysis; diff --git a/Frontend/src/contexts/SessionContext.tsx b/Frontend/src/contexts/SessionContext.tsx new file mode 100644 index 0000000..cf403da --- /dev/null +++ b/Frontend/src/contexts/SessionContext.tsx @@ -0,0 +1,153 @@ +import { createContext, useCallback, useContext, useEffect, useMemo, useState } from "react"; + +import { + TemporarySessionError, + clearTemporarySession, + createTemporarySession, + getStoredSessionToken, + getTemporarySession, + heartbeatTemporarySession, + markTemporarySessionClosing, + removeStoredSessionToken, + storeSessionToken, +} from "@/services/sessionService"; + +type SessionStatus = "initializing" | "active" | "error"; + +interface SessionContextValue { + token: string | null; + expiresAt: number | null; + status: SessionStatus; + error: string | null; + restartSession: () => Promise; +} + +const SessionContext = createContext(undefined); +const HEARTBEAT_MS = 5 * 60 * 1000; + +export const SessionProvider = ({ children }: { children: React.ReactNode }) => { + const [token, setToken] = useState(null); + const [expiresAt, setExpiresAt] = useState(null); + const [status, setStatus] = useState("initializing"); + const [error, setError] = useState(null); + + const startFreshSession = useCallback(async () => { + const session = await createTemporarySession(); + storeSessionToken(session.session_token); + setToken(session.session_token); + setExpiresAt(session.expires_at); + setError(null); + setStatus("active"); + return session.session_token; + }, []); + + const initialize = useCallback(async () => { + setStatus("initializing"); + setError(null); + const storedToken = getStoredSessionToken(); + + if (!storedToken) { + await startFreshSession(); + return; + } + + try { + const session = await getTemporarySession(storedToken); + setToken(storedToken); + setExpiresAt(session.expires_at); + setStatus("active"); + } catch (sessionError) { + if ( + sessionError instanceof TemporarySessionError && + (sessionError.statusCode === 410 || sessionError.code === "SESSION_EXPIRED") + ) { + removeStoredSessionToken(); + await startFreshSession(); + return; + } + throw sessionError; + } + }, [startFreshSession]); + + useEffect(() => { + initialize().catch((sessionError) => { + setStatus("error"); + setError(sessionError instanceof Error ? sessionError.message : "Unable to start a temporary workspace."); + }); + }, [initialize]); + + useEffect(() => { + if (!token) return; + + const heartbeat = async () => { + if (document.visibilityState !== "visible") return; + try { + const session = await heartbeatTemporarySession(token); + setExpiresAt(session.expires_at); + } catch (sessionError) { + if ( + sessionError instanceof TemporarySessionError && + (sessionError.statusCode === 410 || sessionError.code === "SESSION_EXPIRED") + ) { + removeStoredSessionToken(); + setToken(null); + await startFreshSession(); + } + } + }; + + const interval = window.setInterval(heartbeat, HEARTBEAT_MS); + const handleVisible = () => { + if (document.visibilityState === "visible") void heartbeat(); + }; + document.addEventListener("visibilitychange", handleVisible); + + return () => { + window.clearInterval(interval); + document.removeEventListener("visibilitychange", handleVisible); + }; + }, [token, startFreshSession]); + + useEffect(() => { + if (!token) return; + + const handlePageHide = () => { + markTemporarySessionClosing(token); + }; + window.addEventListener("pagehide", handlePageHide); + return () => window.removeEventListener("pagehide", handlePageHide); + }, [token]); + + const restartSession = useCallback(async () => { + const currentToken = getStoredSessionToken(); + removeStoredSessionToken(); + setToken(null); + setExpiresAt(null); + setStatus("initializing"); + setError(null); + + if (currentToken) { + try { + await clearTemporarySession(currentToken); + } catch { + // Explicit restart should still succeed locally if the old session is + // already gone or the cleanup response is interrupted. + } + } + + await startFreshSession(); + }, [startFreshSession]); + + const value = useMemo( + () => ({ token, expiresAt, status, error, restartSession }), + [token, expiresAt, status, error, restartSession], + ); + + return {children}; +}; + +export const useSession = () => { + const context = useContext(SessionContext); + if (!context) throw new Error("useSession must be used within SessionProvider"); + return context; +}; diff --git a/Frontend/src/contexts/TrainingContext.tsx b/Frontend/src/contexts/TrainingContext.tsx index a89c297..db43615 100644 --- a/Frontend/src/contexts/TrainingContext.tsx +++ b/Frontend/src/contexts/TrainingContext.tsx @@ -1,8 +1,25 @@ -import React, { createContext, useContext, useState, useEffect } from 'react'; +import React, { createContext, useContext, useEffect, useRef, useState } from 'react'; + import { useToast } from '@/hooks/use-toast'; import apiService from '@/services/apiService'; -// Run-based architecture types +interface TrainingProgress { + percent: number; + message: string; +} + +interface TrainingResultsSummary { + total_models: number; + successful: number; + failed: number; + best_model?: { + model_type: string; + display_name: string; + metric: string; + value: number; + }; +} + interface TrainingRun { id: string; run_number: number; @@ -10,30 +27,29 @@ interface TrainingRun { started_at?: string; completed_at?: string; duration_seconds?: number; - progress?: { - percent: number; - message: string; - }; - results_summary?: { - total_models: number; - successful: number; - failed: number; - best_model?: { - model_type: string; - display_name: string; - metric: string; - value: number; - }; - }; + progress?: TrainingProgress; + results_summary?: TrainingResultsSummary; error_message?: string; created_at: string; } +interface StartRunResponse { + run_id: string; + run_number: number; + created_at: string; +} + +interface RunStatusResponse { + status: TrainingRun['status']; + progress?: TrainingProgress; + results_summary?: TrainingResultsSummary; + error_message?: string; +} + interface TrainingContextType { currentRun: TrainingRun | null; isTraining: boolean; error: string | null; - startTraining: (experimentId: string) => Promise; stopPolling: () => void; clearTraining: () => void; @@ -49,151 +65,157 @@ export const useTraining = () => { return context; }; +const errorMessage = (error: unknown, fallback: string) => { + if (error instanceof Error && error.message) return error.message; + return fallback; +}; + export const TrainingProvider: React.FC<{ children: React.ReactNode }> = ({ children }) => { const [currentRun, setCurrentRun] = useState(null); const [isTraining, setIsTraining] = useState(false); const [error, setError] = useState(null); - const [pollingInterval, setPollingInterval] = useState(null); - + const pollingIntervalRef = useRef | null>(null); + const { toast } = useToast(); - // Cleanup polling on unmount + const stopPolling = () => { + if (pollingIntervalRef.current) { + clearInterval(pollingIntervalRef.current); + pollingIntervalRef.current = null; + } + setIsTraining(false); + }; + useEffect(() => { return () => { - if (pollingInterval) { - clearInterval(pollingInterval); - setPollingInterval(null); + if (pollingIntervalRef.current) { + clearInterval(pollingIntervalRef.current); + pollingIntervalRef.current = null; } }; - }, []); // Empty deps - only run on unmount + }, []); const startTraining = async (experimentId: string) => { - // Prevent multiple simultaneous training runs if (isTraining) { toast({ - title: "Training in Progress", - description: "Please wait for current training to complete", - variant: "destructive" + title: 'Training in progress', + description: 'Please wait for the current training run to complete.', + variant: 'destructive', }); return; } + stopPolling(); + setError(null); + setIsTraining(true); + try { - setError(null); - setIsTraining(true); - - // Start training run - const response = await apiService.training.startRun(experimentId); - + const response = (await apiService.training.startRun(experimentId)) as StartRunResponse; + setCurrentRun({ id: response.run_id, run_number: response.run_number, status: 'pending', - created_at: response.created_at + created_at: response.created_at, }); - + toast({ - title: "Training Started", - description: `Run #${response.run_number} has been queued`, + title: 'Training started', + description: `Run #${response.run_number} has been queued.`, }); - - // Start polling with retry limit - let retryCount = 0; - const maxRetries = 100; // 5 minutes max (100 * 3 seconds) - + + let pollCount = 0; + let consecutiveErrors = 0; + const maxPolls = 1200; // Up to one hour at a 3 second cadence. + const maxConsecutiveErrors = 8; + const interval = setInterval(async () => { - // Check retry limit - if (retryCount++ > maxRetries) { - clearInterval(interval); - setPollingInterval(null); - setIsTraining(false); - setError('Polling timeout - please refresh to check status'); - toast({ - title: "Polling Timeout", - description: "Training may still be running. Please refresh to check status.", - variant: "destructive" - }); + pollCount += 1; + + if (pollCount > maxPolls) { + stopPolling(); + setError('Training is taking longer than expected. Refresh later to check the run status.'); return; } try { - const status = await apiService.training.getRunStatus(response.run_id); - - setCurrentRun(prev => prev ? { - ...prev, - status: status.status, - progress: status.progress, - error_message: status.error_message, - results_summary: status.results_summary - } : null); - - // Terminal states - stop polling - if (status.status === 'completed' || status.status === 'failed') { - clearInterval(interval); - setPollingInterval(null); - setIsTraining(false); - + const status = (await apiService.training.getRunStatus(response.run_id)) as RunStatusResponse; + consecutiveErrors = 0; + + setCurrentRun((previous) => + previous + ? { + ...previous, + status: status.status, + progress: status.progress, + error_message: status.error_message, + results_summary: status.results_summary, + } + : null, + ); + + if (['completed', 'failed', 'cancelled'].includes(status.status)) { + stopPolling(); + if (status.status === 'completed') { toast({ - title: "Training Complete", - description: `Run #${response.run_number} finished successfully`, + title: 'Training complete', + description: `Run #${response.run_number} finished successfully.`, }); } else { - setError(status.error_message || 'Training failed'); + const detail = status.error_message || (status.status === 'cancelled' ? 'Training was cancelled.' : 'Training failed.'); + setError(detail); toast({ - title: "Training Failed", - description: status.error_message || 'Unknown error', - variant: "destructive", + title: status.status === 'cancelled' ? 'Training cancelled' : 'Training failed', + description: detail, + variant: 'destructive', }); } } - } catch (err) { - console.error('Polling error:', err); - // Don't stop polling on transient errors + } catch (pollError: unknown) { + consecutiveErrors += 1; + console.warn('Training status poll failed', pollError); + + if (consecutiveErrors >= maxConsecutiveErrors) { + stopPolling(); + setError('Lost contact with the training service. Refresh to check whether the run is still active.'); + } } }, 3000); - - setPollingInterval(interval); - - } catch (err: any) { - setIsTraining(false); - setError(err.response?.data?.detail || 'Failed to start training'); + + pollingIntervalRef.current = interval; + } catch (startError: unknown) { + stopPolling(); + const detail = errorMessage(startError, 'Failed to start training.'); + setError(detail); toast({ - title: "Training Failed", - description: err.response?.data?.detail || 'Failed to start training', - variant: "destructive", + title: 'Training failed to start', + description: detail, + variant: 'destructive', }); } }; - const stopPolling = () => { - if (pollingInterval) { - clearInterval(pollingInterval); - setPollingInterval(null); - } - setIsTraining(false); - }; - const clearTraining = () => { stopPolling(); setCurrentRun(null); setError(null); }; - const value: TrainingContextType = { - currentRun, - isTraining, - error, - startTraining, - stopPolling, - clearTraining, - }; - return ( - + {children} ); }; -export default TrainingContext; \ No newline at end of file +export default TrainingContext; diff --git a/Frontend/src/index.css b/Frontend/src/index.css index d29238a..febac2a 100644 --- a/Frontend/src/index.css +++ b/Frontend/src/index.css @@ -2,150 +2,160 @@ @tailwind components; @tailwind utilities; -/* Definition of the design system. All colors, gradients, fonts, etc should be defined here. -All colors MUST be HSL. -*/ - @layer base { :root { - --background: 0 0% 4%; - --foreground: 0 0% 100%; - - --card: 0 0% 10%; - --card-foreground: 0 0% 100%; - - --popover: 0 0% 10%; - --popover-foreground: 0 0% 100%; - - --primary: 180 100% 50%; - --primary-purple: 258 90% 66%; - --primary-blue: 217 91% 60%; - --primary-foreground: 0 0% 4%; - - --secondary: 0 0% 16%; - --secondary-foreground: 0 0% 100%; - - --muted: 0 0% 16%; - --muted-foreground: 0 0% 63%; - - --accent: 258 90% 66%; - --accent-foreground: 0 0% 100%; + --background: 222 47% 5%; + --foreground: 210 40% 98%; - --destructive: 0 84% 60%; - --destructive-foreground: 0 0% 100%; + --card: 222 35% 8%; + --card-foreground: 210 40% 98%; - --border: 0 0% 20%; - --input: 0 0% 16%; - --ring: 180 100% 50%; + --popover: 222 35% 8%; + --popover-foreground: 210 40% 98%; - --success: 160 84% 39%; - --warning: 38 92% 50%; - --info: 217 91% 60%; + --primary: 177 100% 47%; + --primary-purple: 260 92% 68%; + --primary-blue: 215 95% 62%; + --primary-foreground: 222 47% 5%; - --radius: 0.75rem; + --secondary: 220 25% 13%; + --secondary-foreground: 210 40% 98%; - --sidebar-background: 0 0% 98%; + --muted: 220 24% 13%; + --muted-foreground: 215 16% 64%; - --sidebar-foreground: 240 5.3% 26.1%; + --accent: 260 92% 68%; + --accent-foreground: 210 40% 98%; - --sidebar-primary: 240 5.9% 10%; + --destructive: 0 72% 56%; + --destructive-foreground: 210 40% 98%; - --sidebar-primary-foreground: 0 0% 98%; + --border: 217 25% 17%; + --input: 220 24% 13%; + --ring: 177 100% 47%; - --sidebar-accent: 240 4.8% 95.9%; + --success: 158 70% 48%; + --warning: 38 92% 55%; + --info: 215 95% 62%; - --sidebar-accent-foreground: 240 5.9% 10%; + --radius: 0.9rem; - --sidebar-border: 220 13% 91%; - - --sidebar-ring: 217.2 91.2% 59.8%; + --sidebar-background: 222 35% 8%; + --sidebar-foreground: 210 40% 98%; + --sidebar-primary: 177 100% 47%; + --sidebar-primary-foreground: 222 47% 5%; + --sidebar-accent: 220 25% 13%; + --sidebar-accent-foreground: 210 40% 98%; + --sidebar-border: 217 25% 17%; + --sidebar-ring: 177 100% 47%; } .dark { - --background: 0 0% 4%; - --foreground: 0 0% 100%; - - --card: 0 0% 10%; - --card-foreground: 0 0% 100%; - - --popover: 0 0% 10%; - --popover-foreground: 0 0% 100%; - - --primary: 180 100% 50%; - --primary-purple: 258 90% 66%; - --primary-blue: 217 91% 60%; - --primary-foreground: 0 0% 4%; - - --secondary: 0 0% 16%; - --secondary-foreground: 0 0% 100%; - - --muted: 0 0% 16%; - --muted-foreground: 0 0% 63%; - - --accent: 258 90% 66%; - --accent-foreground: 0 0% 100%; - - --destructive: 0 84% 60%; - --destructive-foreground: 0 0% 100%; - - --border: 0 0% 20%; - --input: 0 0% 16%; - --ring: 180 100% 50%; - --sidebar-background: 240 5.9% 10%; - --sidebar-foreground: 240 4.8% 95.9%; - --sidebar-primary: 224.3 76.3% 48%; - --sidebar-primary-foreground: 0 0% 100%; - --sidebar-accent: 240 3.7% 15.9%; - --sidebar-accent-foreground: 240 4.8% 95.9%; - --sidebar-border: 240 3.7% 15.9%; - --sidebar-ring: 217.2 91.2% 59.8%; + --background: 222 47% 5%; + --foreground: 210 40% 98%; + --card: 222 35% 8%; + --card-foreground: 210 40% 98%; + --popover: 222 35% 8%; + --popover-foreground: 210 40% 98%; + --primary: 177 100% 47%; + --primary-purple: 260 92% 68%; + --primary-blue: 215 95% 62%; + --primary-foreground: 222 47% 5%; + --secondary: 220 25% 13%; + --secondary-foreground: 210 40% 98%; + --muted: 220 24% 13%; + --muted-foreground: 215 16% 64%; + --accent: 260 92% 68%; + --accent-foreground: 210 40% 98%; + --destructive: 0 72% 56%; + --destructive-foreground: 210 40% 98%; + --border: 217 25% 17%; + --input: 220 24% 13%; + --ring: 177 100% 47%; + --sidebar-background: 222 35% 8%; + --sidebar-foreground: 210 40% 98%; + --sidebar-primary: 177 100% 47%; + --sidebar-primary-foreground: 222 47% 5%; + --sidebar-accent: 220 25% 13%; + --sidebar-accent-foreground: 210 40% 98%; + --sidebar-border: 217 25% 17%; + --sidebar-ring: 177 100% 47%; } -} -@layer base { * { @apply border-border; } + html { + color-scheme: dark; + scroll-behavior: smooth; + } + body { - @apply bg-background text-foreground antialiased; + @apply min-h-screen bg-background text-foreground antialiased; + background-image: + linear-gradient(to right, hsl(var(--border) / 0.12) 1px, transparent 1px), + linear-gradient(to bottom, hsl(var(--border) / 0.12) 1px, transparent 1px); + background-size: 48px 48px; + font-feature-settings: "rlig" 1, "calt" 1; + } + + ::selection { + background: hsl(var(--primary) / 0.25); + color: hsl(var(--foreground)); + } + + :focus-visible { + outline: 2px solid hsl(var(--ring)); + outline-offset: 2px; } } @layer utilities { .gradient-primary { - background: linear-gradient(135deg, hsl(var(--primary)), hsl(var(--primary-purple)), hsl(var(--primary-blue))); + background: linear-gradient(135deg, hsl(var(--primary)), hsl(var(--primary-blue)) 52%, hsl(var(--primary-purple))); } - + .gradient-text { - background: linear-gradient(135deg, hsl(var(--primary)), hsl(var(--primary-purple)), hsl(var(--primary-blue))); + background: linear-gradient(105deg, hsl(var(--primary)), hsl(var(--primary-blue)) 48%, hsl(var(--primary-purple))); -webkit-background-clip: text; background-clip: text; -webkit-text-fill-color: transparent; } - + .glow-primary { - box-shadow: 0 0 20px hsl(var(--primary) / 0.3); + box-shadow: 0 0 35px hsl(var(--primary) / 0.18); } - + .glow-purple { - box-shadow: 0 0 20px hsl(var(--primary-purple) / 0.3); - } - - .hover-scale { - transition: transform 0.2s ease-in-out; + box-shadow: 0 0 35px hsl(var(--primary-purple) / 0.16); } - - .hover-scale:hover { - transform: scale(1.05); + + .glass-panel { + background: hsl(var(--card) / 0.66); + border: 1px solid hsl(var(--border) / 0.78); + box-shadow: 0 18px 60px hsl(222 60% 2% / 0.35); + backdrop-filter: blur(22px); } - + .card-hover { - transition: all 0.3s ease-in-out; + transition: transform 220ms ease, border-color 220ms ease, box-shadow 220ms ease; } - + .card-hover:hover { - border-color: hsl(var(--primary) / 0.5); - box-shadow: 0 0 30px hsl(var(--primary) / 0.2); + transform: translateY(-3px); + border-color: hsl(var(--primary) / 0.28); + box-shadow: 0 18px 50px hsl(var(--primary) / 0.08); + } +} + +@media (prefers-reduced-motion: reduce) { + *, + *::before, + *::after { + scroll-behavior: auto !important; + animation-duration: 0.01ms !important; + animation-iteration-count: 1 !important; + transition-duration: 0.01ms !important; } } diff --git a/Frontend/src/pages/Datasets.tsx b/Frontend/src/pages/Datasets.tsx index 9ee056c..1c66518 100644 --- a/Frontend/src/pages/Datasets.tsx +++ b/Frontend/src/pages/Datasets.tsx @@ -1,292 +1,324 @@ -import { useState, useEffect } from "react"; -import { Database, Trash2, Eye, Edit2, Plus } from "lucide-react"; -import { Button } from "@/components/ui/button"; -import { Input } from "@/components/ui/input"; -import { AlertDialog, AlertDialogAction, AlertDialogCancel, AlertDialogContent, AlertDialogDescription, AlertDialogFooter, AlertDialogHeader, AlertDialogTitle } from "@/components/ui/alert-dialog"; -import { datasetAPI } from "@/services/apiService"; +import { useCallback, useEffect, useMemo, useState } from "react"; +import { + Clock3, + Columns3, + Database, + Edit2, + Eye, + HardDrive, + Plus, + RotateCcw, + Rows3, + Search, + ShieldCheck, + Trash2, + UploadCloud, +} from "lucide-react"; import { toast } from "sonner"; -import DatasetUploadModal from "@/components/datasets/DatasetUploadModal"; + import DatasetPreviewModal from "@/components/datasets/DatasetPreviewModal"; import DatasetRenameModal from "@/components/datasets/DatasetRenameModal"; +import DatasetUploadModal from "@/components/datasets/DatasetUploadModal"; +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, +} from "@/components/ui/alert-dialog"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { useSession } from "@/contexts/SessionContext"; +import { workspaceDatasetAPI, type WorkspaceDataset } from "@/services/workspaceService"; -// Helper function to format backend dataset response for frontend display -const formatDataset = (dataset: any) => ({ +const toDate = (value: number | string) => { + if (typeof value === "number") return new Date(value < 1_000_000_000_000 ? value * 1000 : value); + return new Date(value); +}; + +const formatDataset = (dataset: WorkspaceDataset) => ({ ...dataset, rows: dataset.row_count, columns: dataset.column_count, - size: dataset.file_size_bytes + size: dataset.file_size_bytes ? `${(dataset.file_size_bytes / (1024 * 1024)).toFixed(2)} MB` - : 'N/A', - uploaded: dataset.created_at - ? new Date(dataset.created_at).toLocaleDateString() - : 'N/A' + : "N/A", + uploaded: dataset.created_at + ? toDate(dataset.created_at).toLocaleString(undefined, { + month: "short", + day: "numeric", + hour: "2-digit", + minute: "2-digit", + }) + : "This session", }); +type DisplayDataset = ReturnType; + +const DatasetSkeleton = () => ( +
    +
    +
    +
    +
    +
    +
    +
    +
    + {[1, 2, 3].map((item) =>
    )} +
    +
    +); + const Datasets = () => { - const [datasets, setDatasets] = useState([]); + const { status: sessionStatus, token, error: sessionError, restartSession } = useSession(); + const [datasets, setDatasets] = useState([]); const [loading, setLoading] = useState(true); const [searchQuery, setSearchQuery] = useState(""); - - // Modals state const [uploadModalOpen, setUploadModalOpen] = useState(false); const [previewModalOpen, setPreviewModalOpen] = useState(false); const [renameModalOpen, setRenameModalOpen] = useState(false); const [deleteDialogOpen, setDeleteDialogOpen] = useState(false); - - // Selected dataset for actions - const [selectedDataset, setSelectedDataset] = useState(null); + const [clearDialogOpen, setClearDialogOpen] = useState(false); + const [selectedDataset, setSelectedDataset] = useState(null); + const [clearing, setClearing] = useState(false); - useEffect(() => { - loadDatasets(); - }, []); - - const loadDatasets = async () => { + const loadDatasets = useCallback(async () => { + if (sessionStatus !== "active") return; setLoading(true); try { - const datasets = await datasetAPI.list(); - // datasetAPI.list() already extracts the array from response - const formattedDatasets = (datasets || []).map(formatDataset); - setDatasets(formattedDatasets); + const response = await workspaceDatasetAPI.list(); + setDatasets(response.map(formatDataset)); } catch (error: any) { - toast.error(error.message || "Failed to load datasets"); + toast.error(error.message || "Failed to load your temporary datasets"); } finally { setLoading(false); } - }; + }, [sessionStatus]); - const [deleteError, setDeleteError] = useState<{ message: string; dependencies: any[] } | null>(null); + useEffect(() => { + if (sessionStatus === "active" && token) void loadDatasets(); + if (sessionStatus === "initializing") setLoading(true); + }, [sessionStatus, token, loadDatasets]); + + const filteredDatasets = useMemo( + () => datasets.filter((dataset) => dataset.name.toLowerCase().includes(searchQuery.trim().toLowerCase())), + [datasets, searchQuery], + ); const handleDelete = async () => { if (!selectedDataset) return; - try { - await datasetAPI.delete(selectedDataset.id); - toast.success("Dataset deleted successfully"); - loadDatasets(); + await workspaceDatasetAPI.delete(selectedDataset.id); + toast.success("Dataset removed from this session"); setDeleteDialogOpen(false); setSelectedDataset(null); - setDeleteError(null); + await loadDatasets(); } catch (error: any) { - // Handle 409 Conflict - dataset has dependencies - if (error.statusCode === 409 && error.dependencies) { - setDeleteError({ - message: error.message, - dependencies: error.dependencies - }); - } else { - toast.error(error.message || "Failed to delete dataset"); - setDeleteDialogOpen(false); - } + toast.error(error.message || "Failed to delete dataset"); } }; - const filteredDatasets = datasets.filter(ds => - ds.name.toLowerCase().includes(searchQuery.toLowerCase()) - ); + const handleClearSession = async () => { + setClearing(true); + try { + await restartSession(); + setDatasets([]); + setSelectedDataset(null); + setSearchQuery(""); + setClearDialogOpen(false); + toast.success("Temporary session cleared. A fresh workspace is ready."); + } catch (error: any) { + toast.error(error.message || "Couldn't reset the temporary session"); + } finally { + setClearing(false); + } + }; return ( -
    -
    -
    -
    +
    +
    +
    +
    +
    +
    +
    + Temporary data workspace +
    +

    + Bring data in. Take results out. +

    +

    + Upload CSV, Excel or Parquet data. NoCodeML keeps it only inside this temporary browser session while you analyze, train and export. +

    +
    + +
    + + +
    +
    +
    + +
    +
    + +
    +

    No permanent dataset storage

    +

    + Dataset files in this workspace are temporary and are deleted when you clear the session or after the session expires. +

    +
    +
    +
    +
    -

    Datasets

    -

    Manage your uploaded datasets and create experiments

    +

    Download before you leave

    +

    + Analysis, chart, model and prediction exports are being added to the final workspace flow so you can keep what matters. +

    -
    +
    + + {sessionStatus === "error" && ( +
    +

    Temporary workspace unavailable

    +

    {sessionError || "Please retry the session."}

    + +
    + )} -
    +
    +
    + setSearchQuery(e.target.value)} + onChange={(event) => setSearchQuery(event.target.value)} + className="h-11 rounded-xl border-border/70 bg-background/55 pl-10 backdrop-blur" />
    +
    + {loading ? "Loading sessionโ€ฆ" : `${filteredDatasets.length} dataset${filteredDatasets.length === 1 ? "" : "s"} in this session`} +
    - + {loading ? ( -
    - {[1, 2, 3].map(i => ( -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    - ))} +
    + {[1, 2, 3].map((item) => )}
    ) : filteredDatasets.length === 0 ? ( -
    -
    - -

    - {searchQuery ? "No datasets found" : "No datasets yet"} -

    -

    - {searchQuery - ? "Try adjusting your search query" - : "Upload your first dataset to start creating ML experiments"} -

    - {!searchQuery && ( - - )} +
    +
    +
    -
    +

    {searchQuery ? "No matching datasets" : "Your temporary ML lab is ready"}

    +

    + {searchQuery + ? "Try another search term or clear the filter." + : "Upload a dataset to inspect it now. Nothing needs to be saved to an account or permanent project database."} +

    + {!searchQuery && ( + + )} + ) : ( -
    +
    {filteredDatasets.map((dataset) => ( -
    -
    -
    -
    - -
    -
    -

    {dataset.name}

    -

    {dataset.uploaded}

    -
    +
    +
    +
    +
    -
    - -
    -
    -

    Rows

    -

    {dataset.rows?.toLocaleString()}

    -
    -
    -

    Columns

    -

    {dataset.columns}

    -
    -
    -

    Size

    -

    {dataset.size}

    +
    +

    {dataset.name}

    +

    {dataset.original_filename}

    +

    Added {dataset.uploaded}

    - -
    -
    - - - -
    + +
    + {[ + { icon: Rows3, label: "Rows", value: dataset.rows?.toLocaleString() ?? "โ€”" }, + { icon: Columns3, label: "Columns", value: dataset.columns ?? "โ€”" }, + { icon: HardDrive, label: "Size", value: dataset.size }, + ].map(({ icon: Icon, label, value }) => ( +
    + +
    {value}
    +
    {label}
    +
    + ))}
    -
    + +
    + + + +
    +
    ))}
    )}
    - {/* Upload Modal */} - + + + - {/* Preview Modal */} - - - {/* Rename Modal */} - + + + + Remove this temporary dataset? + + {selectedDataset?.name} and its uploaded file will be removed from this session. This cannot be undone. + + + + Cancel + void handleDelete()} className="bg-destructive text-destructive-foreground hover:bg-destructive/90">Remove dataset + + + - {/* Delete Confirmation */} - { - setDeleteDialogOpen(open); - if (!open) setDeleteError(null); - }}> - + + - Delete Dataset + Clear the entire session? - {deleteError ? ( -
    -

    {deleteError.message}

    -
    -

    Dependent Experiments:

    -
      - {deleteError.dependencies.map((dep: any) => ( -
    • - โ€ข {dep.name} -
      • - ))} -
    -
    -

    Please delete or reassign these experiments before deleting this dataset.

    -
    - ) : ( - `Are you sure you want to delete "${selectedDataset?.name}"? This action cannot be undone.` - )} + All temporary datasets and any temporary workspace artifacts already created in this session will be deleted. Download anything you need before clearing.     - setDeleteError(null)}> - {deleteError ? 'Close' : 'Cancel'} - - {!deleteError && ( - - Delete - - )} + Keep session + void handleClearSession()} className="bg-destructive text-destructive-foreground hover:bg-destructive/90"> + {clearing ? "Clearingโ€ฆ" : "Clear and start fresh"} +     -
    +
    ); }; diff --git a/Frontend/src/pages/Experiments.tsx b/Frontend/src/pages/Experiments.tsx index 3c43419..3f94980 100644 --- a/Frontend/src/pages/Experiments.tsx +++ b/Frontend/src/pages/Experiments.tsx @@ -1,11 +1,12 @@ -import { useState, useEffect } from "react"; -import { Plus, Search } from "lucide-react"; +import { useEffect, useMemo, useState } from "react"; +import { Beaker, Plus, Search, Sparkles } from "lucide-react"; +import { useNavigate } from "react-router-dom"; + +import CreateExperimentModal from "@/components/experiments/CreateExperimentModal"; +import ExperimentCard from "@/components/experiments/ExperimentCard"; import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; import { useExperiment } from "@/contexts/ExperimentContext"; -import { useNavigate } from "react-router-dom"; -import ExperimentCard from "@/components/experiments/ExperimentCard"; -import CreateExperimentModal from "@/components/experiments/CreateExperimentModal"; import { experimentAPI } from "@/services/apiService"; const Experiments = () => { @@ -15,7 +16,9 @@ const Experiments = () => { const navigate = useNavigate(); useEffect(() => { - fetchExperiments(); + void fetchExperiments(); + // The provider currently recreates this callback as state changes; load once on page entry. + // eslint-disable-next-line react-hooks/exhaustive-deps }, []); const handleCreate = async (name: string, datasetId: string) => { @@ -26,78 +29,116 @@ const Experiments = () => { const handleDuplicate = async (id: string) => { try { await experimentAPI.duplicate(id); - fetchExperiments(); - } catch (error) { - // Error handled in API service + await fetchExperiments(); + } catch { + // The API/context layer surfaces request errors to the user. } }; - const filteredExperiments = experiments.filter(exp => - exp.name.toLowerCase().includes(searchQuery.toLowerCase()) + const filteredExperiments = useMemo( + () => + experiments.filter((experiment) => + experiment.name.toLowerCase().includes(searchQuery.trim().toLowerCase()), + ), + [experiments, searchQuery], ); + const completedCount = experiments.filter((experiment) => experiment.status === "completed").length; + const activeCount = experiments.length - completedCount; + return ( -
    -
    -
    -
    -
    -

    Experiments

    -

    Manage your ML experiments

    +
    +
    +
    +
    +
    + +
    +
    +
    + + Experiment studio +
    +

    + Build, compare and learn +

    +

    + Every run keeps its configuration and results together so you can iterate without losing context. +

    -
    -
    - - setSearchQuery(e.target.value)} - className="pl-10" - /> +
    +
    + + setSearchQuery(event.target.value)} + className="h-11 rounded-xl border-border/70 bg-background/55 pl-10 backdrop-blur" + /> +
    + +
    +
    +
    {experiments.length}
    +
    Total
    +
    +
    +
    {activeCount}
    +
    Active
    +
    +
    +
    {completedCount}
    +
    Completed
    +
    +
    -
    +
    {filteredExperiments.length === 0 ? ( -
    -
    -

    - {searchQuery ? "No experiments found" : "No experiments yet"} -

    -

    - {searchQuery - ? "Try adjusting your search query" - : "Create your first experiment to start training ML models"} -

    - {!searchQuery && ( - - )} +
    +
    +
    -
    +

    + {searchQuery ? "No matching experiments" : "Start your first ML experiment"} +

    +

    + {searchQuery + ? "Try a different search term or clear the filter." + : "Pick a dataset, configure the problem, train multiple models and compare the results in one guided workspace."} +

    + {!searchQuery && ( + + )} + ) : ( -
    - {filteredExperiments.map((exp) => ( +
    + {filteredExperiments.map((experiment) => ( ))} -
    + )}
    @@ -106,7 +147,7 @@ const Experiments = () => { onOpenChange={setCreateModalOpen} onCreate={handleCreate} /> -
    +
    ); }; diff --git a/Frontend/src/pages/Home.tsx b/Frontend/src/pages/Home.tsx index e1e15f6..3e2c665 100644 --- a/Frontend/src/pages/Home.tsx +++ b/Frontend/src/pages/Home.tsx @@ -1,131 +1,105 @@ import { Link } from "react-router-dom"; -import { Upload, BarChart3, Zap, Download, Brain, TrendingUp, Sparkles, MessageSquare } from "lucide-react"; +import { + ArrowRight, + BarChart3, + BrainCircuit, + Download, + ShieldCheck, + Sparkles, + Upload, + WandSparkles, +} from "lucide-react"; + import { Button } from "@/components/ui/button"; const Home = () => { - const features = [ - { - icon: Upload, - title: "Data Upload & Management", - description: "Drag and drop CSV files up to 100MB. Automatic validation and preprocessing suggestions.", - gradient: "from-primary to-primary-blue" - }, - { - icon: BarChart3, - title: "Interactive Visualizations", - description: "Build custom plots with Plotly. Correlation matrices, distributions, and more.", - gradient: "from-primary-purple to-accent" - }, - { - icon: Brain, - title: "Automated Model Training", - description: "Train multiple models simultaneously. Compare performance metrics in real-time.", - gradient: "from-primary-blue to-info" - }, - { - icon: Sparkles, - title: "Smart Hyperparameter Tuning", - description: "Automatic hyperparameter optimization for best model performance without manual configuration.", - gradient: "from-accent to-primary-blue" - }, - { - icon: MessageSquare, - title: "AI Assistant Chatbot", - description: "Get instant help and insights with our integrated AI assistant for data analysis and ML guidance.", - gradient: "from-primary-purple to-primary" - }, - { - icon: TrendingUp, - title: "Real-time Results", - description: "Live training monitoring with detailed evaluation metrics and model comparison.", - gradient: "from-primary to-primary-purple" - }, - { - icon: Zap, - title: "Fast & Intuitive", - description: "Streamlined workflow from data upload to predictions in minutes, not hours.", - gradient: "from-accent to-primary-purple" - } + const workflow = [ + ["Upload", "Bring a CSV, Excel or Parquet dataset."], + ["Explore", "Inspect quality, missing values, distributions and correlations."], + ["Configure", "Pick a target or use NoCodeML's task and model suggestions."], + ["Train", "Compare real classification or regression models."], + ["Predict", "Run single or batch predictions with the fitted pipeline."], + ["Export", "Download charts, metrics, predictions, models or the complete session."], ]; - + return ( -
    - {/* Hero Section */} -
    -
    -
    -
    -

    - Machine Learning Without Code +
    +
    + +
    +
    +
    + No signup. No permanent workspace. +
    + +
    +

    + Machine learning from your data, without the setup.

    -

    - Upload your data, visualize patterns, train models with automated hyperparameter tuning, and get AI-powered insights - all through an intuitive interface +

    + NoCodeML is a temporary, guided AutoML workspace. Upload a dataset, understand it, train and compare models, make predictions and download the results. When the session ends, the workspace is removed.

    -
    - - - - - - -
    -
    -
    - - {/* Features Grid */} -
    -
    -
    -

    - Everything You Need for ML Experiments -

    -

    - From data exploration to automated model optimization with AI assistance, all in one platform -

    + +
    + +
    Guest-first by design
    - -
    - {features.map((feature, index) => ( -
    -
    - -
    -

    {feature.title}

    -

    {feature.description}

    + +
    + {[["8", "ML models"], ["0", "accounts required"], ["1", "guided workspace"]].map(([value, label]) => ( +
    +
    {value}
    +
    {label}
    ))}
    + +
    +
    +
    +
    +

    One clean flow

    From file to useful output

    +
    +
    +
    + {workflow.map(([step, detail], index) => ( +
    +
    {String(index + 1).padStart(2, "0")}
    +
    {step}
    {detail}
    +
    + ))} +
    +
    +
    - - {/* CTA Section */} -
    -
    -
    -

    - Ready to Build Your First Model? -

    -

    - Start experimenting with machine learning in minutes. Powered by automated hyperparameter tuning and AI assistance. No coding required. -

    - - - + +
    +
    + {[ + [Upload, "Temporary uploads", "Files live only inside the active temporary workspace."], + [BarChart3, "EDA before training", "Understand data quality and relationships before choosing a model."], + [WandSparkles, "Smart configuration", "Get task, target, feature and model guidance while keeping control."], + [Download, "Export your work", "Download analysis, metrics, models, predictions and a full session bundle."], + ].map(([Icon, title, description]) => { + const FeatureIcon = Icon as typeof Upload; + return

    {String(title)}

    {String(description)}

    ; + })} +
    +
    + +
    +
    +
    +

    Bring the dataset. Leave with the results.

    No account lifecycle, no saved-project clutter and no permanent visitor database. Download what you need before the temporary session ends.

    +
    -
    +
    ); }; diff --git a/Frontend/src/pages/Login.tsx b/Frontend/src/pages/Login.tsx index b84a787..e3da2fe 100644 --- a/Frontend/src/pages/Login.tsx +++ b/Frontend/src/pages/Login.tsx @@ -1,109 +1,173 @@ -import { useState } from 'react'; -import { Link, useNavigate, useLocation } from 'react-router-dom'; -import { useAuth } from '@/contexts/AuthContext'; -import { Button } from '@/components/ui/button'; -import { Input } from '@/components/ui/input'; -import { Label } from '@/components/ui/label'; -import { Card, CardContent, CardDescription, CardFooter, CardHeader, CardTitle } from '@/components/ui/card'; -import { Activity, Loader2 } from 'lucide-react'; +import { useState } from "react"; +import { ArrowRight, BrainCircuit, Database, Layers3, Loader2, Sparkles } from "lucide-react"; +import { Link, useLocation, useNavigate } from "react-router-dom"; + +import { Button } from "@/components/ui/button"; +import { + Card, + CardContent, + CardDescription, + CardFooter, + CardHeader, + CardTitle, +} from "@/components/ui/card"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { useAuth } from "@/contexts/AuthContext"; const Login = () => { - const [email, setEmail] = useState(''); - const [password, setPassword] = useState(''); + const [email, setEmail] = useState(""); + const [password, setPassword] = useState(""); const [isLoading, setIsLoading] = useState(false); const { login } = useAuth(); const navigate = useNavigate(); const location = useLocation(); - // Get the page user was trying to access before being redirected to login - const from = (location.state as any)?.from?.pathname || '/'; + const from = (location.state as any)?.from?.pathname || "/"; - const handleSubmit = async (e: React.FormEvent) => { - e.preventDefault(); + const handleSubmit = async (event: React.FormEvent) => { + event.preventDefault(); setIsLoading(true); - + try { await login(email, password); - // Redirect to the page they were trying to access, or home navigate(from, { replace: true }); - } catch (error) { - // Error handled by AuthContext + } catch { + // AuthContext surfaces the API error. } finally { setIsLoading(false); } }; return ( -
    -
    -
    -
    - -
    -

    NoCodeML

    -

    Sign in to your account

    -
    - - - - Welcome back - Enter your credentials to access your account - -
    - -
    - - setEmail(e.target.value)} - required - disabled={isLoading} - className="bg-input border-border" - /> +
    +
    +
    + +
    +
    +
    +
    + +
    +
    -
    - - setPassword(e.target.value)} - required - disabled={isLoading} - className="bg-input border-border" - /> + NoCodeML + + +
    +
    + + ML workspace, rebuilt for V3
    - - - -

    - Don't have an account?{' '} - - Sign up - +

    + Turn raw data into decisions. +

    +

    + Explore a dataset, configure an experiment, train multiple models and understand the result without leaving one guided workspace.

    -     - - +
    +
    + +
    + {[ + { icon: Database, title: "Data-first", text: "Profile and inspect before training" }, + { icon: Layers3, title: "Run history", text: "Compare experiments without losing context" }, + ].map(({ icon: Icon, title, text }) => ( +
    + +
    {title}
    +
    {text}
    +
    + ))} +
    +
    + +
    +
    +
    +
    + +
    +

    NoCodeML

    +

    Your guided machine-learning workspace

    +
    + + + +
    Welcome back
    + Sign in to your workspace + Continue your datasets, experiments and model runs. +     + +
    + +
    + + setEmail(event.target.value)} + required + disabled={isLoading} + className="h-11 rounded-xl border-border/70 bg-background/70" + /> +
    +
    + + setPassword(event.target.value)} + required + disabled={isLoading} + className="h-11 rounded-xl border-border/70 bg-background/70" + /> +
    +     + + + + +

    + New to NoCodeML?{" "} + + Create an account + +

    +     +
    +     + +

    + Your experiments and datasets stay scoped to your authenticated NoCodeML account. +

    +
    +
    -
    +
    ); }; diff --git a/Frontend/src/pages/Playground.tsx b/Frontend/src/pages/Playground.tsx index 61c5241..1fbf771 100644 --- a/Frontend/src/pages/Playground.tsx +++ b/Frontend/src/pages/Playground.tsx @@ -1,192 +1,173 @@ -import { useState, useEffect } from "react"; -import { useParams, useNavigate } from "react-router-dom"; -import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; -import { Button } from "@/components/ui/button"; -import { ArrowLeft, Save } from "lucide-react"; -import { useExperiment } from "@/contexts/ExperimentContext"; -import { useEDA } from "@/hooks/useEDA"; +import { useEffect, useState } from "react"; +import { ArrowLeft, Loader2 } from "lucide-react"; +import { useNavigate, useParams } from "react-router-dom"; + +import { DataScienceAssistant } from "@/components/experiments/DataScienceAssistant"; import DataAnalysisStep from "@/components/playground/DataAnalysisStep"; import ModelConfigStep from "@/components/playground/ModelConfigStep"; -import TrainingStep from "@/components/playground/TrainingStep"; -import ResultsStep from "@/components/playground/ResultsStep"; import PredictionStep from "@/components/playground/PredictionStep"; -import { DataScienceAssistant } from "@/components/experiments/DataScienceAssistant"; -import { toast } from "sonner"; +import ResultsStep from "@/components/playground/ResultsStep"; +import TrainingStep from "@/components/playground/TrainingStep"; +import { Button } from "@/components/ui/button"; +import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; +import { useExperiment } from "@/contexts/ExperimentContext"; +import { useTraining } from "@/contexts/TrainingContext"; +import { useEDA } from "@/hooks/useEDA"; + +const phases = [ + { value: "analysis", number: "01", label: "Analysis" }, + { value: "config", number: "02", label: "Configure" }, + { value: "training", number: "03", label: "Train" }, + { value: "results", number: "04", label: "Results" }, + { value: "predict", number: "05", label: "Predict" }, +] as const; + +type Phase = (typeof phases)[number]["value"]; const Playground = () => { - const { experimentId } = useParams(); + const { experimentId } = useParams<{ experimentId: string }>(); const navigate = useNavigate(); - const { currentExperiment, loadExperiment, saveExperimentConfig } = useExperiment(); - const [activeTab, setActiveTab] = useState("analysis"); - const [hasUnsavedChanges, setHasUnsavedChanges] = useState(false); - - // Load EDA data for the experiment's dataset + const { currentExperiment, loadExperiment } = useExperiment(); + const { currentRun } = useTraining(); + const [activeTab, setActiveTab] = useState("analysis"); + const { edaData, isLoading: edaLoading } = useEDA(currentExperiment?.datasetId); - - // Refresh experiment data when switching to config or training tabs to ensure fresh data + useEffect(() => { - if (experimentId && (activeTab === "config" || activeTab === "training")) { - loadExperiment(experimentId); + if (!experimentId) { + navigate("/experiments", { replace: true }); + return; } - }, [activeTab, experimentId]); + void loadExperiment(experimentId); + // loadExperiment is supplied by context and currently recreated by the V2 provider. + // Re-running this effect should depend on the route ID, not provider render identity. + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [experimentId, navigate]); useEffect(() => { - if (experimentId) { - loadExperiment(experimentId); - } else { - navigate("/experiments"); - } - }, [experimentId]); - - const handleSave = async () => { - if (!currentExperiment) return; - - try { - await saveExperimentConfig(currentExperiment.config); - setHasUnsavedChanges(false); - } catch (error) { - // Error handled in context + if (experimentId && (activeTab === "config" || activeTab === "training")) { + void loadExperiment(experimentId); } - }; - - const tabs = [ - { value: "analysis", label: "1. Data Analysis", component: DataAnalysisStep }, - { value: "config", label: "2. Model Config", component: ModelConfigStep }, - { value: "training", label: "3. Training", component: TrainingStep }, - { value: "results", label: "4. Results", component: ResultsStep }, - { value: "predict", label: "5. Prediction", component: PredictionStep } - ]; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [activeTab, experimentId]); if (!currentExperiment) { return ( -
    -
    -

    Loading experiment...

    -

    Please wait

    +
    +
    + + Loading experimentโ€ฆ
    ); } - + + const configurationReady = Boolean( + currentExperiment.config?.taskType && + currentExperiment.config?.targetColumn && + currentExperiment.config?.selectedFeatures?.length, + ); + return ( -
    -
    -
    -
    -
    - -
    -

    {currentExperiment.name}

    -

    - Dataset: {currentExperiment.datasetName || "Loading..."} -

    -
    -
    +
    +
    +
    +
    +
    +
    Experiment workspace
    +

    {currentExperiment.name}

    +

    Dataset: {currentExperiment.datasetName || "Loading datasetโ€ฆ"}

    +
    +
    +
    Status
    +
    {currentExperiment.status.replace("_", " ")}
    +
    -
    - - - - {tabs.map((tab) => ( - - {tab.label} - - ))} - - - - setActiveTab("config")} - /> +

+ + setActiveTab(value as Phase)} className="space-y-5"> +
+ + {phases.map((phase) => ( + + {phase.number} + {phase.label} + + ))} + +
+ + + setActiveTab("config")} /> - - + + {edaLoading ? ( -
-
-
-

Loading dataset information...

-
+
+
Loading dataset analysisโ€ฆ
) : edaData ? ( - setActiveTab("training")} onBack={() => setActiveTab("analysis")} /> ) : ( -
-

Please complete Data Analysis first

- +
+

Complete Data Analysis before configuring models.

+
)} - - - {currentExperiment?.config?.taskType && currentExperiment?.config?.targetColumn && currentExperiment?.config?.selectedFeatures ? ( - + {configurationReady ? ( + { - // Training complete, move to results - setActiveTab("results"); - }} + onComplete={() => setActiveTab("results")} /> ) : ( -
-

Please complete Model Configuration first

- +
+

Choose a task, target, features and models before training.

+
)} - - - setActiveTab("training")} - /> + + + setActiveTab("training")} /> - - + +
- {/* AI Assistant - Always available across all phases */} -
+ ); }; diff --git a/Frontend/src/pages/Register.tsx b/Frontend/src/pages/Register.tsx index bdf72a1..6f6d65c 100644 --- a/Frontend/src/pages/Register.tsx +++ b/Frontend/src/pages/Register.tsx @@ -1,134 +1,225 @@ -import { useState } from 'react'; -import { Link, useNavigate } from 'react-router-dom'; -import { useAuth } from '@/contexts/AuthContext'; -import { Button } from '@/components/ui/button'; -import { Input } from '@/components/ui/input'; -import { Label } from '@/components/ui/label'; -import { Card, CardContent, CardDescription, CardFooter, CardHeader, CardTitle } from '@/components/ui/card'; -import { Activity, Loader2 } from 'lucide-react'; +import { useMemo, useState } from "react"; +import { ArrowRight, BrainCircuit, Check, Loader2, ShieldCheck, Sparkles } from "lucide-react"; +import { Link, useNavigate } from "react-router-dom"; + +import { Button } from "@/components/ui/button"; +import { + Card, + CardContent, + CardDescription, + CardFooter, + CardHeader, + CardTitle, +} from "@/components/ui/card"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { useAuth } from "@/contexts/AuthContext"; const Register = () => { - const [email, setEmail] = useState(''); - const [password, setPassword] = useState(''); - const [confirmPassword, setConfirmPassword] = useState(''); + const [email, setEmail] = useState(""); + const [password, setPassword] = useState(""); + const [confirmPassword, setConfirmPassword] = useState(""); const [isLoading, setIsLoading] = useState(false); - const [error, setError] = useState(''); + const [error, setError] = useState(""); const { register } = useAuth(); const navigate = useNavigate(); - const handleSubmit = async (e: React.FormEvent) => { - e.preventDefault(); - setError(''); + const passwordChecks = useMemo( + () => [ + { label: "8+ characters", passed: password.length >= 8 }, + { label: "Passwords match", passed: Boolean(password) && password === confirmPassword }, + ], + [confirmPassword, password], + ); - if (password !== confirmPassword) { - setError('Passwords do not match'); - return; - } + const handleSubmit = async (event: React.FormEvent) => { + event.preventDefault(); + setError(""); if (password.length < 8) { - setError('Password must be at least 8 characters'); + setError("Password must be at least 8 characters."); + return; + } + if (password !== confirmPassword) { + setError("Passwords do not match."); return; } setIsLoading(true); - try { await register(email, password); - navigate('/login'); - } catch (error) { - // Error handled by AuthContext + navigate("/login"); + } catch { + // AuthContext surfaces API errors. } finally { setIsLoading(false); } }; return ( -
-
-
-
- -
-

NoCodeML

-

Create your account

-
- - - - Get started - Create an account to start building ML models - -
- -
- - setEmail(e.target.value)} - required - disabled={isLoading} - className="bg-input border-border" - /> +
+
+
+ +
+
+
+
+
+
-
- - setPassword(e.target.value)} - required - disabled={isLoading} - className="bg-input border-border" - /> +

NoCodeML

+

Start building ML experiments without the boilerplate

+
+ + + +
Create your workspace
+ Start experimenting + Create an account and move from raw data to model comparison in one guided flow. + + + + +
+ + setEmail(event.target.value)} + required + disabled={isLoading} + className="h-11 rounded-xl border-border/70 bg-background/70" + /> +
+ +
+ + setPassword(event.target.value)} + required + disabled={isLoading} + className="h-11 rounded-xl border-border/70 bg-background/70" + /> +
+ +
+ + setConfirmPassword(event.target.value)} + required + disabled={isLoading} + className="h-11 rounded-xl border-border/70 bg-background/70" + /> +
+ +
+ {passwordChecks.map((check) => ( + + + {check.label} + + ))} +
+ + {error && ( +
+ {error} +
+ )} + + + + + +

+ Already have an account?{" "} + + Sign in + +

+ + + +
+
+ +
+
+
+ +
+
-
- - setConfirmPassword(e.target.value)} - required - disabled={isLoading} - className="bg-input border-border" - /> + NoCodeML + + +
+
+ + Guided AutoML workspace
- {error && ( -

{error}

- )} - - - -

- Already have an account?{' '} - - Sign in - +

+ Learn by building real models. +

+

+ Keep the important ML decisions visible: data quality, feature choices, model configuration, metrics and predictions.

- - - +
+
+ +
+
+
+ +
+
+
Isolated workspace
+

+ Your NoCodeML account, datasets and experiments stay inside the dedicated application database scope. +

+
+
+
+
-
+
); }; diff --git a/Frontend/src/pages/Workspace.tsx b/Frontend/src/pages/Workspace.tsx new file mode 100644 index 0000000..4039b90 --- /dev/null +++ b/Frontend/src/pages/Workspace.tsx @@ -0,0 +1,433 @@ +import { useCallback, useEffect, useMemo, useState } from "react"; +import Plot from "react-plotly.js"; +import { + ArrowLeft, + ArrowRight, + BarChart3, + BrainCircuit, + CheckCircle2, + Download, + FileDown, + Loader2, + Play, + RefreshCw, + ShieldCheck, + Sparkles, + Target, + UploadCloud, + WandSparkles, +} from "lucide-react"; +import { toast } from "sonner"; + +import DatasetUploadModal from "@/components/datasets/DatasetUploadModal"; +import DataReadinessPanel from "@/components/playground/DataReadinessPanel"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { Input } from "@/components/ui/input"; +import { useSession } from "@/contexts/SessionContext"; +import { + workspaceDatasetAPI, + workspaceEDAAPI, + workspaceExportAPI, + workspacePredictionAPI, + workspaceTrainingAPI, + type WorkspaceDataset, + type WorkspacePlotResponse, + type WorkspacePrediction, + type WorkspaceTrainingRun, +} from "@/services/workspaceService"; +import type { EDAResponse } from "@/types/experiment"; + +const STEPS = ["Data", "Explore", "Configure", "Train", "Predict & Export"] as const; +const CLASSIFICATION_MODELS = [ + ["LogisticRegression", "Logistic Regression", "Fast, interpretable baseline"], + ["RandomForestClassifier", "Random Forest", "Strong nonlinear ensemble"], + ["XGBClassifier", "XGBoost", "High-performance gradient boosting"], + ["LGBMClassifier", "LightGBM", "Efficient boosted trees"], +] as const; +const REGRESSION_MODELS = [ + ["LinearRegression", "Linear Regression", "Fast, interpretable baseline"], + ["RandomForestRegressor", "Random Forest", "Robust nonlinear ensemble"], + ["XGBRegressor", "XGBoost", "High-performance gradient boosting"], + ["LGBMRegressor", "LightGBM", "Efficient boosted trees"], +] as const; + +const selectClass = "h-11 w-full rounded-xl border border-border/70 bg-background/60 px-3 text-sm outline-none transition focus:border-primary/60 focus:ring-2 focus:ring-primary/10"; + +const slug = (value: string) => value.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "").slice(0, 64) || "dataset"; +const chartFilename = (dataset: string, kind: string) => { + const now = new Date(); + const stamp = `${now.getFullYear()}${String(now.getMonth() + 1).padStart(2, "0")}${String(now.getDate()).padStart(2, "0")}-${String(now.getHours()).padStart(2, "0")}${String(now.getMinutes()).padStart(2, "0")}${String(now.getSeconds()).padStart(2, "0")}`; + return `nocodeml_${slug(dataset)}_${slug(kind)}_${stamp}`; +}; + +const Workspace = () => { + const { status: sessionStatus, error: sessionError, restartSession } = useSession(); + const [step, setStep] = useState(0); + const [datasets, setDatasets] = useState([]); + const [datasetId, setDatasetId] = useState(""); + const [uploadOpen, setUploadOpen] = useState(false); + const [loadingDatasets, setLoadingDatasets] = useState(true); + const [eda, setEda] = useState(null); + const [edaLoading, setEdaLoading] = useState(false); + + const [target, setTarget] = useState(""); + const [taskType, setTaskType] = useState<"classification" | "regression">("classification"); + const [features, setFeatures] = useState([]); + const [selectedModels, setSelectedModels] = useState([]); + const [testSize, setTestSize] = useState(0.2); + + const [run, setRun] = useState(null); + const [startingTraining, setStartingTraining] = useState(false); + const [predictionValues, setPredictionValues] = useState>({}); + const [predictionResult, setPredictionResult] = useState | null>(null); + const [predicting, setPredicting] = useState(false); + const [batchFile, setBatchFile] = useState(null); + const [batchPredicting, setBatchPredicting] = useState(false); + const [predictionHistory, setPredictionHistory] = useState([]); + + const [plotType, setPlotType] = useState<"histogram" | "scatter" | "box" | "correlation" | "bar">("histogram"); + const [plotX, setPlotX] = useState(""); + const [plotY, setPlotY] = useState(""); + const [plotGroup, setPlotGroup] = useState(""); + const [plotData, setPlotData] = useState(null); + const [plotLoading, setPlotLoading] = useState(false); + + const activeDataset = useMemo(() => datasets.find((dataset) => dataset.id === datasetId) || null, [datasets, datasetId]); + const modelOptions = taskType === "classification" ? CLASSIFICATION_MODELS : REGRESSION_MODELS; + + const loadDatasets = useCallback(async () => { + if (sessionStatus !== "active") return; + setLoadingDatasets(true); + try { + const next = await workspaceDatasetAPI.list(); + setDatasets(next); + setDatasetId((current) => next.some((dataset) => dataset.id === current) ? current : (next[0]?.id || "")); + } catch (error) { + toast.error(error instanceof Error ? error.message : "Couldn't load this temporary workspace"); + } finally { + setLoadingDatasets(false); + } + }, [sessionStatus]); + + useEffect(() => { + if (sessionStatus === "active") void loadDatasets(); + }, [sessionStatus, loadDatasets]); + + useEffect(() => { + if (!datasetId) { + setEda(null); + return; + } + let cancelled = false; + setEdaLoading(true); + workspaceEDAAPI.summary(datasetId) + .then((summary) => { if (!cancelled) setEda(summary); }) + .catch((error) => { if (!cancelled) toast.error(error instanceof Error ? error.message : "Couldn't analyze this dataset"); }) + .finally(() => { if (!cancelled) setEdaLoading(false); }); + return () => { cancelled = true; }; + }, [datasetId]); + + useEffect(() => { + if (!eda) return; + const ids = new Set(eda.id_columns || []); + const candidates = eda.columns.filter((column) => !ids.has(column.name) && column.unique_count > 1); + const hints = ["target", "label", "outcome", "class", "churn", "survived", "fraud", "default", "price", "sales", "revenue", "score"]; + const suggested = candidates.find((column) => hints.some((hint) => column.name.toLowerCase().includes(hint))) || candidates[candidates.length - 1]; + if (!suggested) return; + + setTarget(suggested.name); + const numeric = new Set(eda.numeric_columns || []); + const categorical = new Set(eda.categorical_columns || []); + const lowCardinality = suggested.unique_count <= Math.max(20, Math.floor(eda.dataset_info.row_count * 0.05)); + const inferred: "classification" | "regression" = categorical.has(suggested.name) || (numeric.has(suggested.name) && lowCardinality) ? "classification" : "regression"; + setTaskType(inferred); + setFeatures(eda.columns.filter((column) => column.name !== suggested.name && !ids.has(column.name) && column.unique_count > 1).map((column) => column.name)); + setSelectedModels(inferred === "classification" ? ["LogisticRegression", "RandomForestClassifier"] : ["LinearRegression", "RandomForestRegressor"]); + setRun(null); + setPredictionResult(null); + setPredictionValues({}); + + const numericColumns = (eda.numeric_columns || []).filter((column) => !ids.has(column)); + const categoricalColumns = (eda.categorical_columns || []).filter((column) => !ids.has(column)); + setPlotX(numericColumns[0] || categoricalColumns[0] || ""); + setPlotY(numericColumns[1] || ""); + setPlotGroup(""); + setPlotData(null); + }, [eda]); + + useEffect(() => { + if (!target || !eda) return; + const column = eda.columns.find((item) => item.name === target); + if (!column) return; + const categorical = new Set(eda.categorical_columns || []); + const numeric = new Set(eda.numeric_columns || []); + const lowCardinality = column.unique_count <= Math.max(20, Math.floor(eda.dataset_info.row_count * 0.05)); + const inferred: "classification" | "regression" = categorical.has(target) || (numeric.has(target) && lowCardinality) ? "classification" : "regression"; + setTaskType(inferred); + setFeatures((current) => current.filter((feature) => feature !== target)); + setSelectedModels(inferred === "classification" ? ["LogisticRegression", "RandomForestClassifier"] : ["LinearRegression", "RandomForestRegressor"]); + setRun(null); + }, [target, eda]); + + useEffect(() => { + if (!run || !["queued", "running"].includes(run.status)) return; + const timer = window.setInterval(async () => { + try { + const next = await workspaceTrainingAPI.get(run.id); + setRun(next); + if (next.status === "completed") { + toast.success("Training complete"); + window.clearInterval(timer); + } else if (next.status === "failed") { + toast.error(next.error?.message || "Training failed"); + window.clearInterval(timer); + } + } catch (error) { + toast.error(error instanceof Error ? error.message : "Couldn't refresh training status"); + } + }, 1000); + return () => window.clearInterval(timer); + }, [run]); + + useEffect(() => { + if (step === 4 && sessionStatus === "active") { + workspacePredictionAPI.list().then(setPredictionHistory).catch(() => setPredictionHistory([])); + } + }, [step, sessionStatus]); + + const generatePlot = async () => { + if (!datasetId || !eda) return; + if (plotType !== "correlation" && !plotX) return toast.error("Choose a column for this chart"); + if (plotType === "scatter" && !plotY) return toast.error("Choose both X and Y columns for a scatter plot"); + setPlotLoading(true); + try { + setPlotData(await workspaceEDAAPI.plot(datasetId, { + plot_type: plotType, + x_column: plotType === "correlation" ? "unused" : plotX, + y_column: plotType === "scatter" ? plotY : null, + group_by: plotGroup || null, + })); + } catch (error) { + toast.error(error instanceof Error ? error.message : "Couldn't generate this visualization"); + } finally { + setPlotLoading(false); + } + }; + + const startTraining = async () => { + if (!datasetId || !target || !features.length || !selectedModels.length) return toast.error("Choose a target, at least one feature and at least one model"); + setStartingTraining(true); + try { + const next = await workspaceTrainingAPI.start({ + dataset_id: datasetId, + target_column: target, + task_type: taskType, + selected_features: features, + models: selectedModels.map((model_type) => ({ model_type })), + test_size: testSize, + random_state: 42, + cv_folds: 3, + scaling: true, + }); + setRun(next); + toast.success("Training started"); + } catch (error) { + toast.error(error instanceof Error ? error.message : "Training couldn't start"); + } finally { + setStartingTraining(false); + } + }; + + const makePrediction = async () => { + if (!run || run.status !== "completed") return; + const numeric = new Set(eda?.numeric_columns || []); + const payload: Record = {}; + for (const feature of features) { + const raw = predictionValues[feature]?.trim(); + if (!raw) return toast.error(`Enter a value for ${feature}`); + const value = numeric.has(feature) ? Number(raw) : raw; + if (numeric.has(feature) && Number.isNaN(value)) return toast.error(`${feature} must be a number`); + payload[feature] = value; + } + setPredicting(true); + try { + setPredictionResult(await workspaceTrainingAPI.predict(run.id, payload)); + } catch (error) { + toast.error(error instanceof Error ? error.message : "Prediction failed"); + } finally { + setPredicting(false); + } + }; + + const makeBatchPrediction = async () => { + if (!run || !batchFile) return; + setBatchPredicting(true); + try { + const prediction = await workspaceTrainingAPI.predictBatch(run.id, batchFile); + toast.success(`${prediction.total_predictions} predictions generated`); + setBatchFile(null); + setPredictionHistory(await workspacePredictionAPI.list()); + } catch (error) { + toast.error(error instanceof Error ? error.message : "Batch prediction failed"); + } finally { + setBatchPredicting(false); + } + }; + + const clearWorkspace = async () => { + if (!window.confirm("Clear this temporary session? Download anything you need first. All workspace files will be deleted.")) return; + try { + await restartSession(); + setDatasets([]); + setDatasetId(""); + setEda(null); + setRun(null); + setPredictionHistory([]); + setStep(0); + toast.success("Fresh temporary workspace ready"); + } catch (error) { + toast.error(error instanceof Error ? error.message : "Couldn't reset the workspace"); + } + }; + + if (sessionStatus === "error") { + return

Workspace unavailable

{sessionError || "NoCodeML couldn't start a temporary session."}

; + } + + const canContinue = step === 0 ? Boolean(datasetId) : step === 1 ? Boolean(eda) : step === 2 ? Boolean(target && features.length && selectedModels.length) : step === 3 ? run?.status === "completed" : false; + + return ( +
+
+
+
+
+
No account ยท Temporary by design
+

Temporary ML workspace

+

Upload, explore, train, predict and export. NoCodeML removes your workspace after you leave or the session expires.

+
+
+ + +
+
+
+ +
+ {STEPS.map((label, index) => ( + + ))} +
+ + {step === 0 && ( + + Choose your data + + {loadingDatasets || sessionStatus === "initializing" ? ( +
Preparing temporary workspaceโ€ฆ
+ ) : datasets.length ? ( +
+ {datasets.map((dataset) => ( + + ))} +
+ ) : ( +

Drop in a dataset to begin

CSV, Excel and Parquet are supported up to 100 MB. No signup and no permanent project record.

+ )} + + + + )} + + {step === 1 && ( +
+ {edaLoading || !eda ? Analyzing datasetโ€ฆ : ( + <> + + + Download analysis + + + + + + + + + Visualization lab + +
+ + + + +
+ {(plotType === "scatter" || plotType === "box") && } + {plotData &&
} + + + + )} +
+ )} + + {step === 2 && eda && ( +
+ + What should NoCodeML predict? + +
+
Suggested task

The suggestion uses target datatype and cardinality. You can override it.

+
setTestSize(Number(event.target.value))} className="w-full accent-primary" />
+ + + + Features & models + +
Input features{features.length} selected
{eda.columns.filter((column) => column.name !== target && !eda.id_columns.includes(column.name) && column.unique_count > 1).map((column) => { const checked = features.includes(column.name); return ; })}
+
ModelsSmart defaults selected
{modelOptions.map(([value, label, description]) => { const checked = selectedModels.includes(value); return ; })}
+ + +
+ )} + + {step === 3 && ( + + Train & compare + + {!run &&
Task
{taskType}
Target
{target}
Models
{selectedModels.length}
} + {run &&
{run.status}

{run.progress.message}

{run.progress.percent}%
{run.status === "failed" &&
{run.error?.message || "Training failed. Review the configuration and try again."}
}
} + {run?.status === "completed" &&
Best model
{String(run.best_model?.model_type || "Model")}
{String(run.best_model?.metric || "score")}: {Number(run.best_model?.score ?? 0).toFixed(4)}
{run.results.map((result, index) => { const metrics = (result.metrics as { test?: Record } | undefined)?.test || {}; const primary = taskType === "classification" ? metrics.f1_score : metrics.r2_score; return ; })}
ModelStatusPrimary metricTime
{String(result.model_type)}{result.success ? "Completed" : "Failed"}{typeof primary === "number" ? primary.toFixed(4) : "โ€”"}{typeof result.training_time_seconds === "number" ? `${result.training_time_seconds.toFixed(2)}s` : "โ€”"}
} + + + )} + + {step === 4 && run?.status === "completed" && eda && ( +
+
+ Single prediction{features.map((feature) => { const numeric = eda.numeric_columns.includes(feature); const column = eda.columns.find((item) => item.name === feature); return
setPredictionValues((current) => ({ ...current, [feature]: event.target.value }))} placeholder={column?.sample_values?.length ? `e.g. ${String(column.sample_values[0])}` : "Enter a value"} className="rounded-xl" />
; })}{predictionResult &&
Prediction
{String(predictionResult.prediction)}
{typeof predictionResult.confidence === "number" &&
Confidence: {(predictionResult.confidence * 100).toFixed(1)}%
}
} + Batch prediction

Upload a CSV containing the same feature columns used for training. The generated prediction file remains temporary until you download it.

setBatchFile(event.target.files?.[0] || null)} /> +
+ Your downloadable outputs{predictionHistory.length ? predictionHistory.map((prediction) =>
{prediction.download_filename}
{prediction.total_predictions.toLocaleString()} predictions
) :

Batch prediction downloads will appear here.

}
 +
+ )} + +
+ + {step < STEPS.length - 1 && } +
+
+ + void loadDatasets()} /> +
+ ); +}; + +export default Workspace; diff --git a/Frontend/src/services/sessionService.ts b/Frontend/src/services/sessionService.ts new file mode 100644 index 0000000..6848df3 --- /dev/null +++ b/Frontend/src/services/sessionService.ts @@ -0,0 +1,103 @@ +const SESSION_STORAGE_KEY = "nocodeml_temporary_session"; + +const rawApiUrl = import.meta.env.VITE_API_URL; +const API_BASE_URL = rawApiUrl && /^https?:\/\//i.test(rawApiUrl.trim()) + ? rawApiUrl.trim().replace(/\/$/, "") + : "http://localhost:8000"; + +export interface TemporarySession { + session_token: string; + expires_at: number; + ttl_seconds: number; + close_grace_seconds?: number; + temporary: boolean; + privacy?: string; + status?: string; +} + +export class TemporarySessionError extends Error { + statusCode?: number; + code?: string; + + constructor(message: string, statusCode?: number, code?: string) { + super(message); + this.name = "TemporarySessionError"; + this.statusCode = statusCode; + this.code = code; + } +} + +const parseResponse = async (response: Response): Promise => { + if (response.ok) { + if (response.status === 204) return undefined as T; + return response.json() as Promise; + } + + let payload: { detail?: { code?: string; message?: string } | string } | undefined; + try { + payload = await response.json(); + } catch { + payload = undefined; + } + + const detail = payload?.detail; + const message = typeof detail === "string" + ? detail + : detail?.message || "The temporary NoCodeML workspace is unavailable."; + const code = typeof detail === "object" ? detail?.code : undefined; + throw new TemporarySessionError(message, response.status, code); +}; + +export const getStoredSessionToken = () => sessionStorage.getItem(SESSION_STORAGE_KEY); + +export const storeSessionToken = (token: string) => { + sessionStorage.setItem(SESSION_STORAGE_KEY, token); +}; + +export const removeStoredSessionToken = () => { + sessionStorage.removeItem(SESSION_STORAGE_KEY); +}; + +export const createTemporarySession = async (): Promise => { + const response = await fetch(`${API_BASE_URL}/api/v1/session`, { + method: "POST", + headers: { Accept: "application/json" }, + }); + return parseResponse(response); +}; + +export const getTemporarySession = async (token: string): Promise => { + const response = await fetch(`${API_BASE_URL}/api/v1/session`, { + headers: { + Accept: "application/json", + "X-NoCodeML-Session": token, + }, + }); + return parseResponse(response); +}; + +export const heartbeatTemporarySession = async (token: string): Promise => { + const response = await fetch(`${API_BASE_URL}/api/v1/session/heartbeat`, { + method: "POST", + headers: { + Accept: "application/json", + "X-NoCodeML-Session": token, + }, + }); + return parseResponse(response); +}; + +export const clearTemporarySession = async (token: string): Promise => { + const response = await fetch(`${API_BASE_URL}/api/v1/session`, { + method: "DELETE", + headers: { "X-NoCodeML-Session": token }, + }); + await parseResponse(response); +}; + +export const markTemporarySessionClosing = (token: string) => { + const form = new URLSearchParams({ session_token: token }); + return navigator.sendBeacon(`${API_BASE_URL}/api/v1/session/end`, form); +}; + +export { API_BASE_URL, SESSION_STORAGE_KEY }; diff --git a/Frontend/src/services/workspaceService.ts b/Frontend/src/services/workspaceService.ts new file mode 100644 index 0000000..70f192f --- /dev/null +++ b/Frontend/src/services/workspaceService.ts @@ -0,0 +1,281 @@ +import type { EDAResponse } from "@/types/experiment"; +import { + API_BASE_URL, + TemporarySessionError, + getStoredSessionToken, + removeStoredSessionToken, +} from "@/services/sessionService"; + +export class WorkspaceApiError extends Error { + statusCode?: number; + code?: string; + + constructor(message: string, statusCode?: number, code?: string) { + super(message); + this.name = "WorkspaceApiError"; + this.statusCode = statusCode; + this.code = code; + } +} + +const requireSession = () => { + const token = getStoredSessionToken(); + if (!token) { + throw new TemporarySessionError("Your temporary workspace is still starting. Please try again.", 428, "SESSION_REQUIRED"); + } + return token; +}; + +const errorFromResponse = async (response: Response) => { + let payload: { detail?: string | { code?: string; message?: string } } | undefined; + try { + payload = await response.json(); + } catch { + payload = undefined; + } + + const detail = payload?.detail; + const code = typeof detail === "object" ? detail?.code : undefined; + const message = typeof detail === "string" + ? detail + : detail?.message || "NoCodeML couldn't complete that workspace action."; + + if (response.status === 410 || code === "SESSION_EXPIRED") removeStoredSessionToken(); + return new WorkspaceApiError(message, response.status, code); +}; + +const request = async (path: string, init: RequestInit = {}): Promise => { + const token = requireSession(); + const headers = new Headers(init.headers); + headers.set("Accept", "application/json"); + headers.set("X-NoCodeML-Session", token); + + const response = await fetch(`${API_BASE_URL}${path}`, { ...init, headers }); + if (response.ok) { + if (response.status === 204) return undefined as T; + return response.json() as Promise; + } + throw await errorFromResponse(response); +}; + +const parseDownloadName = (response: Response, fallback: string) => { + const disposition = response.headers.get("content-disposition") || ""; + const utf8 = disposition.match(/filename\*=UTF-8''([^;]+)/i)?.[1]; + if (utf8) { + try { return decodeURIComponent(utf8.replace(/^"|"$/g, "")); } catch { /* fall through */ } + } + const basic = disposition.match(/filename="?([^";]+)"?/i)?.[1]; + return basic || fallback; +}; + +const download = async (path: string, fallbackFilename: string) => { + const token = requireSession(); + const response = await fetch(`${API_BASE_URL}${path}`, { + headers: { "X-NoCodeML-Session": token }, + }); + if (!response.ok) throw await errorFromResponse(response); + + const blob = await response.blob(); + const url = URL.createObjectURL(blob); + const anchor = document.createElement("a"); + anchor.href = url; + anchor.download = parseDownloadName(response, fallbackFilename); + document.body.appendChild(anchor); + anchor.click(); + anchor.remove(); + URL.revokeObjectURL(url); + return anchor.download; +}; + +export interface WorkspaceDataset { + id: string; + name: string; + description?: string | null; + original_filename: string; + file_size_bytes: number; + row_count: number; + column_count: number; + column_info: unknown; + created_at: number | string; + temporary: true; +} + +export interface WorkspaceDatasetPreview { + columns: string[]; + data: unknown[][]; + row_count: number; + preview_rows: number; +} + +export interface WorkspacePlotRequest { + plot_type: "histogram" | "scatter" | "box" | "correlation" | "bar"; + x_column: string; + y_column?: string | null; + group_by?: string | null; +} + +export interface WorkspacePlotResponse { + data: Record[]; + layout: Record; + is_sampled: boolean; + total_rows: number; + displayed_rows: number; + plot_type: string; +} + +export interface WorkspaceTrainingModel { + model_type: string; + hyperparameters?: Record; + enable_optimization?: boolean; +} + +export interface WorkspaceTrainingRequest { + dataset_id: string; + target_column: string; + task_type: "classification" | "regression"; + selected_features?: string[] | null; + models: WorkspaceTrainingModel[]; + test_size?: number; + random_state?: number; + cv_folds?: number; + scaling?: boolean; +} + +export interface WorkspaceTrainingRun { + id: string; + dataset_id: string; + dataset_name: string; + status: "queued" | "running" | "completed" | "failed"; + progress: { + current: number; + total: number; + percent: number; + current_model?: string | null; + message: string; + }; + config: Record; + results: Array>; + best_model?: Record | null; + error?: { code?: string; message?: string } | null; + created_at: number; + updated_at?: number; + started_at?: number; + completed_at?: number; + temporary: true; +} + +export interface WorkspacePrediction { + id: string; + run_id: string; + dataset_id?: string; + dataset_name: string; + model_type?: string; + download_filename: string; + download_url: string; + total_predictions: number; + created_at: number; + temporary: true; +} + +export const workspaceDatasetAPI = { + upload: async (file: File, name?: string, description?: string) => { + const form = new FormData(); + form.append("file", file); + if (name) form.append("name", name); + if (description) form.append("description", description); + const response = await request<{ dataset: WorkspaceDataset }>("/api/v1/workspace/datasets", { + method: "POST", + body: form, + }); + return response.dataset; + }, + + list: async () => { + const response = await request<{ datasets: WorkspaceDataset[]; total: number }>("/api/v1/workspace/datasets"); + return response.datasets; + }, + + get: async (datasetId: string) => { + const response = await request<{ dataset: WorkspaceDataset }>(`/api/v1/workspace/datasets/${datasetId}`); + return response.dataset; + }, + + preview: (datasetId: string, rows = 10) => + request(`/api/v1/workspace/datasets/${datasetId}/preview?rows=${rows}`), + + update: async (datasetId: string, data: { name: string; description?: string | null }) => { + const response = await request<{ dataset: WorkspaceDataset }>(`/api/v1/workspace/datasets/${datasetId}`, { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(data), + }); + return response.dataset; + }, + + delete: (datasetId: string) => request(`/api/v1/workspace/datasets/${datasetId}`, { method: "DELETE" }), +}; + +export const workspaceEDAAPI = { + summary: (datasetId: string) => request(`/api/v1/workspace/datasets/${datasetId}/eda`), + plot: (datasetId: string, payload: WorkspacePlotRequest) => + request(`/api/v1/workspace/datasets/${datasetId}/plot`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(payload), + }), + downloadSummary: (datasetId: string) => download(`/api/v1/workspace/datasets/${datasetId}/export/summary`, "nocodeml_eda-summary.json"), + downloadStatistics: (datasetId: string) => download(`/api/v1/workspace/datasets/${datasetId}/export/statistics`, "nocodeml_statistics.csv"), + downloadMissingValues: (datasetId: string) => download(`/api/v1/workspace/datasets/${datasetId}/export/missing-values`, "nocodeml_missing-values.csv"), + downloadCorrelations: (datasetId: string) => download(`/api/v1/workspace/datasets/${datasetId}/export/correlations`, "nocodeml_correlations.csv"), +}; + +export const workspaceTrainingAPI = { + start: (payload: WorkspaceTrainingRequest) => + request("/api/v1/workspace/training/runs", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(payload), + }), + get: (runId: string) => request(`/api/v1/workspace/training/runs/${runId}`), + list: async () => { + const response = await request<{ runs: WorkspaceTrainingRun[]; total: number }>("/api/v1/workspace/training/runs"); + return response.runs; + }, + downloadSummary: (runId: string) => download(`/api/v1/workspace/training/runs/${runId}/export/summary`, "nocodeml_training-summary.json"), + downloadComparison: (runId: string) => download(`/api/v1/workspace/training/runs/${runId}/export/model-comparison`, "nocodeml_model-comparison.csv"), + downloadFeatureImportance: (runId: string) => download(`/api/v1/workspace/training/runs/${runId}/export/feature-importance`, "nocodeml_feature-importance.csv"), + downloadBestModel: (runId: string) => download(`/api/v1/workspace/training/runs/${runId}/export/best-model`, "nocodeml_best-model.joblib"), + predict: (runId: string, features: Record) => + request<{ + prediction: unknown; + probabilities?: Record | null; + confidence?: number | null; + model_type?: string; + run_id: string; + temporary: true; + }>(`/api/v1/workspace/training/runs/${runId}/predict`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ features }), + }), + predictBatch: (runId: string, file: File) => { + const form = new FormData(); + form.append("file", file); + return request(`/api/v1/workspace/training/runs/${runId}/predict/batch`, { + method: "POST", + body: form, + }); + }, +}; + +export const workspacePredictionAPI = { + list: async () => { + const response = await request<{ predictions: WorkspacePrediction[]; total: number }>("/api/v1/workspace/predictions"); + return response.predictions; + }, + download: (prediction: WorkspacePrediction) => download(prediction.download_url, prediction.download_filename), +}; + +export const workspaceExportAPI = { + downloadSession: () => download("/api/v1/workspace/export/session", "nocodeml_session.zip"), +}; diff --git a/Frontend/src/types/experiment.ts b/Frontend/src/types/experiment.ts index 235e08f..1f72d42 100644 --- a/Frontend/src/types/experiment.ts +++ b/Frontend/src/types/experiment.ts @@ -1,5 +1,3 @@ -// Frontend/src/types/experiment.ts - export interface FeatureTypes { numerical: string[]; categorical: string[]; @@ -8,9 +6,9 @@ export interface FeatureTypes { export interface ModelConfig { model_type: string; display_name: string; - preset: "fast" | "balanced" | "accurate"; - hyperparameters: Record; - custom_hyperparameters?: Record | null; + preset: "fast" | "default"; + hyperparameters: Record; + custom_hyperparameters?: Record | null; } export interface ExperimentConfig { @@ -23,16 +21,16 @@ export interface ExperimentConfig { randomSeed?: number; models?: ModelConfig[]; enableOptimization?: boolean; - - // Deprecated fields (backward compatibility) + + // Deprecated V2 compatibility fields. New V3 code should not write these. features?: string[]; selectedModels?: string[]; } export interface AppliedRule { parameter: string; - original_value: any; - value: any; + original_value: unknown; + value: unknown; reason: string; } @@ -52,7 +50,7 @@ export interface HyperparameterTuning { test_score: number; applied_rules: AppliedRule[]; dataset_info: DatasetInfo; - best_params: Record; + best_params: Record; } export interface ExperimentResponse { @@ -62,7 +60,7 @@ export interface ExperimentResponse { datasetName?: string; status: "in_progress" | "completed"; config: ExperimentConfig; - results?: Record; + results?: Record; createdAt: string; updatedAt?: string; } @@ -74,7 +72,18 @@ export interface ColumnInfo { missing_percent: number; unique_count: number; is_id_column: boolean; - sample_values?: any[]; + sample_values?: unknown[]; +} + +export interface NumericColumnStatistics { + count: number; + mean: number; + std: number; + min: number; + "25%": number; + "50%": number; + "75%": number; + max: number; } export interface EDAResponse { @@ -91,7 +100,7 @@ export interface EDAResponse { numeric_columns: string[]; categorical_columns: string[]; id_columns: string[]; - statistics: Record; + statistics: Record; correlations: { columns: string[]; matrix: number[][]; @@ -113,7 +122,7 @@ export interface EDAResponse { }; preview_data: { columns: string[]; - rows: Array>; + rows: Array>; total_rows: number; page_size: number; }; diff --git a/Frontend/tailwind.config.ts b/Frontend/tailwind.config.ts index cc3e4cd..5387ec0 100644 --- a/Frontend/tailwind.config.ts +++ b/Frontend/tailwind.config.ts @@ -1,4 +1,5 @@ import type { Config } from "tailwindcss"; +import tailwindcssAnimate from "tailwindcss-animate"; export default { darkMode: ["class"], @@ -70,49 +71,25 @@ export default { }, keyframes: { "accordion-down": { - from: { - height: "0", - }, - to: { - height: "var(--radix-accordion-content-height)", - }, + from: { height: "0" }, + to: { height: "var(--radix-accordion-content-height)" }, }, "accordion-up": { - from: { - height: "var(--radix-accordion-content-height)", - }, - to: { - height: "0", - }, + from: { height: "var(--radix-accordion-content-height)" }, + to: { height: "0" }, }, "fade-in": { - "0%": { - opacity: "0", - transform: "translateY(10px)" - }, - "100%": { - opacity: "1", - transform: "translateY(0)" - } + "0%": { opacity: "0", transform: "translateY(10px)" }, + "100%": { opacity: "1", transform: "translateY(0)" }, }, "slide-up": { - "0%": { - opacity: "0", - transform: "translateY(20px)" - }, - "100%": { - opacity: "1", - transform: "translateY(0)" - } + "0%": { opacity: "0", transform: "translateY(20px)" }, + "100%": { opacity: "1", transform: "translateY(0)" }, }, "pulse-glow": { - "0%, 100%": { - opacity: "1" - }, - "50%": { - opacity: "0.5" - } - } + "0%, 100%": { opacity: "1" }, + "50%": { opacity: "0.5" }, + }, }, animation: { "accordion-down": "accordion-down 0.2s ease-out", @@ -123,5 +100,5 @@ export default { }, }, }, - plugins: [require("tailwindcss-animate")], + plugins: [tailwindcssAnimate], } satisfies Config; diff --git a/Frontend/vercel.json b/Frontend/vercel.json new file mode 100644 index 0000000..d5031a9 --- /dev/null +++ b/Frontend/vercel.json @@ -0,0 +1,33 @@ +{ + "$schema": "https://openapi.vercel.sh/vercel.json", + "framework": "vite", + "rewrites": [ + { + "source": "/(.*)", + "destination": "/index.html" + } + ], + "headers": [ + { + "source": "/(.*)", + "headers": [ + { + "key": "X-Content-Type-Options", + "value": "nosniff" + }, + { + "key": "X-Frame-Options", + "value": "DENY" + }, + { + "key": "Referrer-Policy", + "value": "strict-origin-when-cross-origin" + }, + { + "key": "Permissions-Policy", + "value": "camera=(), microphone=(), geolocation=()" + } + ] + } + ] +} diff --git a/README.md b/README.md index bffa8ee..4863b2f 100644 --- a/README.md +++ b/README.md @@ -1,419 +1,373 @@ -# NoCodeML Platform ๐Ÿš€ +# NoCodeML V3 + +> Upload a dataset, understand it, train and compare machine-learning models, make predictions, download the outputs, and leave. No account required and no permanent visitor workspace. + +![React](https://img.shields.io/badge/React-18-61DAFB?logo=react&logoColor=white) +![TypeScript](https://img.shields.io/badge/TypeScript-5-3178C6?logo=typescript&logoColor=white) +![FastAPI](https://img.shields.io/badge/FastAPI-Python_3.11-009688?logo=fastapi&logoColor=white) +![scikit-learn](https://img.shields.io/badge/scikit--learn-ML-F7931E?logo=scikitlearn&logoColor=white) +![CI](https://img.shields.io/badge/GitHub_Actions-CI-2088FF?logo=githubactions&logoColor=white) + +## What V3 is + +NoCodeML V3 is a guest-first AutoML workspace built for a simple lifecycle: + +```text +Open NoCodeML + โ†“ +Upload dataset + โ†“ +Explore + ML Readiness + โ†“ +Choose target / task / features / models + โ†“ +Train + compare + โ†“ +Predict + โ†“ +Download useful outputs + โ†“ +Leave / clear session + โ†“ +Temporary workspace removed +``` -![Python](https://img.shields.io/badge/Python-3.11+-blue) -![FastAPI](https://img.shields.io/badge/FastAPI-0.100+-green) -![React](https://img.shields.io/badge/React-18-blue) -![TypeScript](https://img.shields.io/badge/TypeScript-5.0+-blue) +There is **no mandatory signup or login** in the public V3 workflow. ---- +Visitor datasets, generated models, training state, predictions and exports are **not written to PostgreSQL/Supabase**. They live only inside an isolated temporary session workspace on the backend. -## ๐Ÿ“ธ Preview +## Privacy-by-lifecycle design -![NoCodeML Platform Landing Page](./screenshots/landing-page.png) -*NoCodeML Platform - Your gateway to no-code machine learning* +Each browser session receives a cryptographically random token. The raw token is never used as a server directory name; NoCodeML stores the workspace under a SHA-256 digest of the token. ---- +A workspace contains only temporary folders such as: -## ๐ŸŽฏ What We Built +```text +/tmp/nocodeml-sessions// +โ”œโ”€โ”€ datasets/ +โ”œโ”€โ”€ analysis/ +โ”œโ”€โ”€ training/ +โ”œโ”€โ”€ models/ +โ”œโ”€โ”€ predictions/ +โ””โ”€โ”€ exports/ +``` -NoCodeML is an end-to-end machine learning platform that removes the coding barrier from ML development. Users can: -- Upload datasets (CSV, Excel, Parquet) -- Perform automated exploratory data analysis with interactive visualizations -- Select and configure ML models from 8 powerful algorithms -- Train models asynchronously with real-time progress tracking -- Make predictions on new data -- Compare model performance across different algorithms +Cleanup has multiple layers: -**Tech Stack:** FastAPI + React + PostgreSQL + Redis + Celery + Docker -**Models:** 4 Classification + 4 Regression (Logistic/Linear, Random Forest, XGBoost, LightGBM) +- **Clear & restart** deletes the current workspace immediately. +- Browser close/navigation sends a best-effort cleanup signal with a short grace period so normal refreshes do not destroy work accidentally. +- Inactive sessions expire automatically (60 minutes by default). +- A cleanup loop removes expired/orphaned workspaces. +- Active ML jobs hold a temporary cleanup lease so files are not deleted halfway through training. -### Why This Matters -Machine learning shouldn't require a CS degree. NoCodeML makes ML accessible to: -- Business analysts exploring data patterns -- Students learning ML concepts -- Researchers testing hypotheses quickly -- Anyone with data and questions to answer +A browser cannot guarantee that a final network request is delivered when a tab or laptop disappears unexpectedly, so inactivity expiry is the hard cleanup fallback. ---- +## Guided workspace -## ๐Ÿš€ Quick Start +The production UI is intentionally one coherent flow instead of an account/project CRUD dashboard. -### Prerequisites -- Docker & Docker Compose -- Node.js 18+ (with npm or bun) +### 1. Data -### Setup (5 minutes) +- CSV, Excel (`.xlsx` / `.xls`) and Parquet uploads. +- Server-side 100 MB upload limit. +- Safe generated storage names; original filenames do not control server paths. +- Row/column counts and metadata. +- Dataset selection within the current temporary session. -```bash -# 1. Clone and navigate -git clone -cd V2_NoCodeML +### 2. Explore -# 2. Backend - Start all services (API, PostgreSQL, Redis, Celery Worker) -cd Backend -cp .env.example .env -docker-compose up -d -docker-compose exec fastapi_app alembic upgrade head +- ML Readiness score. +- Missing-value analysis. +- Descriptive statistics. +- Conservative ID-column detection. +- Correlations. +- Histogram, scatter, box, bar and correlation visualizations. +- Chart PNG export through Plotly with readable NoCodeML filenames. +- EDA JSON, statistics CSV, missing-values CSV and correlations CSV downloads. -# 3. Frontend - Install and run -cd ../Frontend -npm install # or: bun install -cp .env.example .env -npm run dev # or: bun run dev -``` +### 3. Configure -### Access Points -- **Application:** http://localhost:5173 -- **API Docs:** http://localhost:8000/docs -- **Backend:** http://localhost:8000 - ---- - -## โœจ Core Features - -### 1๏ธโƒฃ Dataset Management -- **Multi-format Support:** Upload CSV, Excel (.xlsx/.xls), and Parquet files -- **Smart Metadata Extraction:** Automatic detection of data types, row/column counts -- **Quick Preview:** View dataset samples directly in the browser -- **Full CRUD Operations:** Rename, update, delete, and manage multiple datasets - -### 2๏ธโƒฃ Exploratory Data Analysis (EDA) -- **Automated Statistics:** Mean, median, std deviation, quartiles for all numeric columns -- **Missing Value Analysis:** Identify and visualize data quality issues -- **Correlation Matrix:** Interactive heatmap showing feature relationships -- **Smart Visualizations:** - - Distribution plots (histograms) - - Box plots for outlier detection - - Scatter plots for bivariate analysis - - Powered by Plotly (backend) and Recharts (frontend) -- **Outlier Detection:** IQR-based statistical outlier identification - -### 3๏ธโƒฃ Experiment Workflow -- **Project Organization:** Create experiments linked to specific datasets -- **Version Control:** Duplicate experiments to test different configurations -- **Configuration Persistence:** Save and load model configurations -- **5-Step Guided Workflow:** Analysis โ†’ Config โ†’ Training โ†’ Results โ†’ Prediction - -### 4๏ธโƒฃ Machine Learning Models (8 Algorithms) - -**Classification** (4 models) -- **Logistic Regression** - Fast linear classifier for binary/multi-class problems -- **Random Forest** - Ensemble of decision trees with feature importance -- **XGBoost** - Gradient boosting with regularization and high accuracy -- **LightGBM** - Ultra-fast gradient boosting optimized for speed - -**Regression** (4 models) -- **Linear Regression** - Simple and interpretable linear model -- **Random Forest** - Robust ensemble regressor with feature importance -- **XGBoost** - High-accuracy gradient boosting for regression -- **LightGBM** - Memory-efficient and fast gradient boosting - -### 5๏ธโƒฃ Model Training System -- **Asynchronous Processing:** Celery-based distributed task queue -- **Real-time Status:** Live progress tracking with job status updates -- **Multi-model Training:** Train multiple algorithms simultaneously -- **Smart Hyperparameters:** Pre-configured optimal defaults for each model -- **Comprehensive Metrics:** - - **Classification:** Accuracy, Precision, Recall, F1-Score, ROC-AUC - - **Regression:** Rยฒ, MAE, MSE, RMSE -- **Model Persistence:** Automatic saving of trained models with joblib -- **Feature Importance:** Supported for Random Forest, XGBoost, and LightGBM models - -### 6๏ธโƒฃ Prediction Engine -- **Single Predictions:** Real-time predictions on individual data points -- **Batch Processing:** Upload CSV files for bulk predictions -- **Result Export:** Download predictions as CSV files -- **Confidence Scores:** Probability estimates for classification tasks - -### 7๏ธโƒฃ User Experience -- **Authentication:** JWT-based secure user system -- **Modern UI:** Built with shadcn/ui components and Tailwind CSS -- **Responsive Design:** Works seamlessly on desktop and mobile -- **Interactive Charts:** Dynamic, zoomable visualizations -- **Real-time Feedback:** Toast notifications and progress indicators - ---- - -## ๐Ÿ—๏ธ Technical Architecture - -### Backend Stack -| Component | Technology | Purpose | -|-----------|-----------|---------| -| API Framework | FastAPI (Python 3.11) | High-performance async REST API | -| Database | PostgreSQL 15 | Persistent storage for users, datasets, experiments | -| ORM | SQLAlchemy (async) | Database abstraction layer | -| Migrations | Alembic | Database schema version control | -| Task Queue | Celery + Redis | Asynchronous ML model training | -| ML Libraries | scikit-learn, XGBoost, LightGBM | 8 optimized ML algorithms | -| Data Processing | pandas, numpy, openpyxl, pyarrow | Dataset handling and transformations | -| Visualization | Plotly | Server-side chart generation | -| Authentication | JWT (python-jose) | Secure user authentication | -| Containerization | Docker + Docker Compose | Easy deployment and scaling | - -### Frontend Stack -| Component | Technology | Purpose | -|-----------|-----------|---------| -| Framework | React 18 + TypeScript | Type-safe component-based UI | -| Build Tool | Vite | Fast development and optimized builds | -| UI Library | shadcn/ui (Radix UI) | Accessible, customizable components | -| Styling | Tailwind CSS | Utility-first responsive design | -| State Management | React Context API | Global state for auth, experiments | -| Data Fetching | TanStack Query | Caching and server state management | -| Routing | React Router v6 | Client-side navigation | -| Charts | Recharts | Interactive data visualizations | - -### System Architecture +NoCodeML provides editable guidance for: -``` -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ Client Browser โ”‚ -โ”‚ (React + TypeScript) โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ - โ”‚ HTTP/REST - โ–ผ -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ FastAPI Backend โ”‚ -โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ -โ”‚ โ”‚ Auth API โ”‚ โ”‚ Dataset API โ”‚ โ”‚ Training API โ”‚ โ”‚ -โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ - โ”‚ โ”‚ โ”‚ - โ–ผ โ–ผ โ–ผ -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ PostgreSQL โ”‚ โ”‚ Redis โ”‚ โ”‚ Celery Worker โ”‚ -โ”‚ (Database) โ”‚ โ”‚ (Task Queue) โ”‚ โ”‚ (ML Training) โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ -``` +- likely target columns; +- classification vs regression; +- usable features; +- train/test split; +- suitable model defaults. + +Users can override those suggestions when domain knowledge says otherwise. + +### 4. Train & compare + +Eight models are supported: + +| Classification | Regression | +| --- | --- | +| Logistic Regression | Linear Regression | +| Random Forest Classifier | Random Forest Regressor | +| XGBoost Classifier | XGBoost Regressor | +| LightGBM Classifier | LightGBM Regressor | -### Project Structure +The guest release uses a **bounded in-process training pool** instead of requiring Celery/Redis infrastructure. By default only one training run is active per session and global worker count is intentionally small for safe free/small deployments. +Training writes status, metrics and model artifacts only into the active temporary workspace. + +Downloads include: + +- model comparison CSV; +- training summary JSON; +- feature importance CSV; +- best fitted model (`.joblib`). + +### 5. Correct preprocessing and inference + +Training builds a fitted scikit-learn pipeline containing: + +- median imputation for numerical features; +- optional numerical scaling; +- most-frequent imputation for categorical features; +- `OneHotEncoder(handle_unknown="ignore")`; +- the estimator; +- fitted classification label decoder where needed. + +Prediction reuses that exact fitted pipeline. NoCodeML does **not** create a new category encoder from prediction input. + +### 6. Predict & export + +- Single-row predictions. +- Classification probability/confidence where available. +- Batch CSV prediction. +- Downloadable prediction CSVs with readable filenames. +- Complete session ZIP export. + +Example names: + +```text +nocodeml_customer-churn_statistics_20260822-171400.csv +nocodeml_customer-churn_feature-importance_20260822-171400.csv +nocodeml_customer-churn_predictions_20260822-171400.csv +nocodeml_customer-churn_best-model_20260822-171400.joblib +nocodeml_customer-churn_session_20260822-171400.zip ``` -V2_NoCodeML/ -โ”œโ”€โ”€ Backend/ # FastAPI backend application -โ”‚ โ”œโ”€โ”€ app/ -โ”‚ โ”‚ โ”œโ”€โ”€ api/ # API route handlers -โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ auth.py # Authentication endpoints -โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ datasets.py # Dataset management -โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ eda.py # Exploratory data analysis -โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ experiments.py # Experiment CRUD -โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ models.py # ML model catalog -โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ training.py # Model training -โ”‚ โ”‚ โ”‚ โ””โ”€โ”€ predictions.py # Prediction endpoints -โ”‚ โ”‚ โ”œโ”€โ”€ core/ # Core configuration -โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ config.py # App settings -โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ security.py # Auth utilities -โ”‚ โ”‚ โ”‚ โ””โ”€โ”€ model_cache.py # Model caching -โ”‚ โ”‚ โ”œโ”€โ”€ db/ # Database configuration -โ”‚ โ”‚ โ”œโ”€โ”€ models/ # SQLAlchemy ORM models -โ”‚ โ”‚ โ”œโ”€โ”€ schemas/ # Pydantic validation schemas -โ”‚ โ”‚ โ”œโ”€โ”€ services/ # Business logic layer -โ”‚ โ”‚ โ”œโ”€โ”€ worker/ # Celery tasks -โ”‚ โ”‚ โ””โ”€โ”€ main.py # Application entry point -โ”‚ โ”œโ”€โ”€ alembic/ # Database migrations -โ”‚ โ”œโ”€โ”€ requirements.txt # Python dependencies -โ”‚ โ”œโ”€โ”€ Dockerfile -โ”‚ โ””โ”€โ”€ docker-compose.yaml -โ”‚ -โ””โ”€โ”€ Frontend/ # React frontend application - โ”œโ”€โ”€ src/ - โ”‚ โ”œโ”€โ”€ components/ # Reusable UI components - โ”‚ โ”‚ โ”œโ”€โ”€ datasets/ # Dataset components - โ”‚ โ”‚ โ”œโ”€โ”€ experiments/ # Experiment components - โ”‚ โ”‚ โ”œโ”€โ”€ playground/ # ML workflow components - โ”‚ โ”‚ โ””โ”€โ”€ ui/ # shadcn/ui components - โ”‚ โ”œโ”€โ”€ contexts/ # React Context providers - โ”‚ โ”œโ”€โ”€ hooks/ # Custom React hooks - โ”‚ โ”œโ”€โ”€ pages/ # Page components - โ”‚ โ”œโ”€โ”€ services/ # API services - โ”‚ โ””โ”€โ”€ types/ # TypeScript type definitions - โ”œโ”€โ”€ package.json - โ””โ”€โ”€ vite.config.ts + +## Data Science Assistant + +The floating assistant is optional and server-side. + +It is authenticated by the temporary session token, not by a user account. The assistant context is deliberately built from **derived workspace information** such as dataset shape, configuration, metrics and model results. Raw uploaded dataset rows are not automatically injected into provider requests. + +If `GEMINI_API_KEY` is not configured, the assistant fails safely while the core ML workflow continues to work. + +## Architecture + +```text +โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” +โ”‚ React + TypeScript + Vite โ”‚ +โ”‚ Data โ†’ Explore โ†’ Configure โ†’ Train โ†’ Predict & Export โ”‚ +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ + โ”‚ X-NoCodeML-Session + โ–ผ +โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” +โ”‚ FastAPI โ”‚ +โ”‚ Session API ยท Workspace API ยท Models ยท Optional AI proxy โ”‚ +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ + โ”‚ + โ–ผ + isolated temporary session folder + datasets / analysis / models / exports + โ”‚ + โ–ผ + bounded in-process ML pool + +No visitor PostgreSQL database +No visitor account store +No Redis/Celery runtime requirement +No persistent application volume required ``` ---- +## Technology stack -## ๐Ÿ› ๏ธ Development Guide +| Layer | Technology | +| --- | --- | +| Frontend | React 18, TypeScript, Vite, React Router | +| UI | Tailwind CSS, shadcn/ui, Radix UI, Lucide | +| Charts | Plotly | +| Backend | FastAPI, Pydantic, HTTPX | +| ML | scikit-learn, XGBoost, LightGBM | +| Data | pandas, NumPy, PyArrow, OpenPyXL | +| Model format | joblib | +| Optional AI | server-side Gemini | +| Local runtime | Docker / Docker Compose | +| Quality | TypeScript, ESLint, pytest, GitHub Actions | -### Backend Commands +## Supabase / legacy database note -```bash -# Start all services (API, PostgreSQL, Redis, Celery) -cd Backend -docker-compose up -d +During the V3 revival an isolated `nocodeml` schema was created under the shared Project Hub. It remains documented and isolated, but **the guest-first V3 runtime does not use it for visitor work**. -# View logs -docker-compose logs -f fastapi_app -docker-compose logs -f celery_worker +The project safety documents remain in the repository: -# Database migrations with Alembic -docker-compose exec fastapi_app alembic upgrade head # Apply all migrations -docker-compose exec fastapi_app alembic revision --autogenerate -m "add_column" # Create new migration -docker-compose exec fastapi_app alembic downgrade -1 # Rollback last migration -docker-compose exec fastapi_app alembic current # Show current revision -docker-compose exec fastapi_app alembic history # Show migration history +- [`AGENTS.md`](./AGENTS.md) +- [`SUPABASE_HUB_RULES.md`](./SUPABASE_HUB_RULES.md) -# Access container shell -docker-compose exec fastapi_app bash +Those rules still prohibit cross-project database access. The old migration/auth/persistence implementation is preserved in Git history and checkpoint branches rather than exposed by the public V3 API. -# Stop all services -docker-compose down -``` +## Public API surface -### Frontend Commands +The public release intentionally mounts only non-persistent routes: -```bash -cd Frontend +| Area | Endpoint examples | +| --- | --- | +| Temporary session | `POST /api/v1/session`, `POST /api/v1/session/heartbeat`, `DELETE /api/v1/session` | +| Temporary datasets | `POST /api/v1/workspace/datasets`, preview / EDA / plot / delete routes | +| Temporary training | `POST /api/v1/workspace/training/runs`, `GET /api/v1/workspace/training/runs/{id}` | +| Prediction | single + batch workspace prediction routes | +| Exports | EDA, training, prediction and complete-session downloads | +| Models | `GET /api/v1/models` | +| Optional AI | `POST /api/v1/assistant/chat` | + +Legacy `/auth`, persistent `/datasets`, `/experiments`, persistent `/training` and persistent `/predictions` routes are not mounted in the guest release. -# Development server (hot reload) -npm run dev # or: bun run dev +FastAPI documentation is available at `/docs` while the backend is running. -# Production build -npm run build # or: bun run build +## Local development -# Preview production build -npm run preview +### Backend -# Lint code -npm run lint +```bash +git clone https://github.com/Rishikeshsanin/NoCodeML.git +cd NoCodeML +git switch release/v3-revival +cd Backend +cp .env.example .env +docker compose up --build ``` -### Environment Configuration +Backend: -**Backend** (`Backend/.env`): -```env -POSTGRES_USER=myuser -POSTGRES_PASSWORD=mysecretpassword -POSTGRES_DB=nocodeml_db -DATABASE_URL=postgresql+psycopg://myuser:mysecretpassword@postgres:5432/nocodeml_db -CELERY_BROKER_URL=redis://redis:6379/0 -CELERY_RESULT_BACKEND=redis://redis:6379/0 -SECRET_KEY=your-secret-key-change-in-production +```text +API: http://localhost:8000 +Docs: http://localhost:8000/docs +Health: http://localhost:8000/health +Ready: http://localhost:8000/ready ``` -**Frontend** (`Frontend/.env`): -```env -VITE_API_URL=http://localhost:8000 +The V3 Compose file runs only the API and uses tmpfs for visitor workspaces. It does not start Postgres, Redis or Celery. + +### Frontend + +```bash +cd Frontend +cp .env.example .env +npm ci +npm run dev ``` ---- +```text +http://localhost:5173 +``` -## ๐Ÿ“ก API Reference +Frontend environment: -Full interactive API documentation: **http://localhost:8000/docs** +```env +VITE_API_URL=http://localhost:8000 +``` -### Core Endpoints +`VITE_*` values are public browser configuration. Never put private provider credentials there. -| Category | Endpoint | Method | Description | -|----------|----------|--------|-------------| -| **Auth** | `/api/v1/auth/register` | POST | Register new user | -| | `/api/v1/auth/login` | POST | Login and get JWT token | -| **Datasets** | `/api/v1/datasets` | GET, POST | List/upload datasets | -| | `/api/v1/datasets/{id}` | GET, PUT, DELETE | Manage dataset | -| | `/api/v1/datasets/{id}/preview` | GET | Preview dataset rows | -| **Experiments** | `/api/v1/experiments` | GET, POST | List/create experiments | -| | `/api/v1/experiments/{id}` | GET, PUT, DELETE | Manage experiment | -| | `/api/v1/experiments/{id}/duplicate` | POST | Duplicate experiment | -| **EDA** | `/api/v1/eda/{dataset_id}/summary` | GET | Get statistics & correlations | -| | `/api/v1/eda/plot` | POST | Generate visualization | -| **Models** | `/api/v1/models` | GET | List all 8 ML models | -| | `/api/v1/models/{task_type}` | GET | Filter by classification/regression | -| **Training** | `/api/v1/training/start` | POST | Start async training jobs | -| | `/api/v1/training/runs/{job_id}` | GET | Get training status & results | -| **Predictions** | `/api/v1/predictions/batch` | POST | Batch predictions from CSV | -| | `/api/v1/predictions/{batch_id}` | GET | Get prediction results | +Backend production environment is intentionally small: ---- +```env +ENVIRONMENT=production +SESSION_ROOT_DIR=/tmp/nocodeml-sessions +SESSION_TTL_MINUTES=60 +SESSION_CLEANUP_INTERVAL_SECONDS=300 +SESSION_CLOSE_GRACE_SECONDS=30 +WORKSPACE_TRAINING_WORKERS=1 +WORKSPACE_MAX_MODELS_PER_RUN=8 +BACKEND_CORS_ORIGINS=https://your-frontend.example +GEMINI_API_KEY=optional-server-side-key +GEMINI_MODEL=gemini-3.7-flash +``` + +No `DATABASE_URL`, PostgreSQL, Supabase, Redis or Celery service is required for the public guest workflow. -## ๐Ÿ—„๏ธ Database Schema & Migrations +## Automated validation -### Alembic Migrations +GitHub Actions validates: -The project uses **Alembic** for database schema version control, ensuring smooth schema evolution across environments. +### Frontend -#### Current Migrations +```text +npm ci +TypeScript typecheck +Vite production build +ESLint +critical npm vulnerability audit +``` -1. **`001_create_datasets_table.py`** - User datasets table - - Stores uploaded dataset metadata (name, file path, rows, columns) - - User-scoped with foreign key to users table +### Backend + +```text +Python compile +full pytest suite +guest-session isolation and cleanup +real classification workflow +real regression workflow +single/batch prediction +export/download behavior +production Docker image +non-root container runtime +/health + /ready + temporary-session container smoke test +ARM64 ML compatibility +``` -2. **`002_create_experiments_table.py`** - ML experiments table - - Links experiments to datasets - - Stores experiment configuration (features, target, task type) +The real ML tests cover mixed numerical/categorical data, persisted preprocessing, unseen categories, classification and regression. -3. **`003_create_training_tables.py`** - Training infrastructure - - Training jobs table (status, model type, hyperparameters) - - Training results table (metrics, feature importance) +## Error and resource guardrails -4. **`004_create_training_runs_table.py`** - Enhanced training tracking - - Detailed job execution tracking - - Training logs and progress monitoring +The application includes controlled handling for malformed/empty uploads, unsupported file types, oversized files, invalid targets/features, model failures, prediction schema mismatches, expired sessions and unavailable AI. -5. **`005_create_prediction_batches_table.py`** - Prediction system - - Batch prediction management - - Prediction results storage +Resource defaults are intentionally conservative for public college-project hosting: -#### Migration Commands +- upload size: **100 MB**; +- idle session: **60 minutes**; +- active training runs per session: **1**; +- global training pool: **bounded**; +- models per run: **up to 8**. -```bash -# Initialize database (first time setup) -docker-compose exec fastapi_app alembic upgrade head +## Repository branches -# Create new migration after model changes -docker-compose exec fastapi_app alembic revision --autogenerate -m "add_new_column" +| Branch | Purpose | +| --- | --- | +| `main` | Original V2 until the V3 release is merged | +| `legacy/v2-2026-08-22` | Permanent V2 recovery point | +| `checkpoint/v3-rc-persistent-2026-08-22` | Working persistence-based V3 RC before guest refactor | +| `release/v3-revival` | Guest-first V3 release candidate | -# Apply migrations -docker-compose exec fastapi_app alembic upgrade head +## Release checklist -# Rollback last migration -docker-compose exec fastapi_app alembic downgrade -1 +- [x] Preserve V2 and the persistence-based V3 checkpoint +- [x] Repair ML preprocessing/inference correctness +- [x] ML Readiness + Smart target/task/model guidance +- [x] Anonymous temporary sessions +- [x] Session isolation and automatic cleanup +- [x] Database-free dataset + EDA workflow +- [x] Database-free classification/regression training +- [x] Temporary single + batch prediction +- [x] Download/export engine and readable filenames +- [x] Guest-first routing with no signup wall +- [x] Guest-session AI assistant +- [x] Remove persistent routes from the public API +- [x] Remove Postgres/Redis/Celery from the public runtime architecture +- [ ] Green final CI on the release head +- [ ] Production backend deployment +- [ ] Vercel frontend deployment +- [ ] Live classification + regression E2E QA +- [ ] Merge to `main` +- [ ] Tag `v3.0.0` -# View current database version -docker-compose exec fastapi_app alembic current +## Project philosophy -# View migration history -docker-compose exec fastapi_app alembic history --verbose -``` +**Quality > quantity.** -### Database Tables - -| Table | Purpose | Key Columns | -|-------|---------|-------------| -| `users` | User authentication | email, hashed_password, created_at | -| `datasets` | Uploaded datasets | name, file_path, num_rows, num_columns, user_id | -| `experiments` | ML experiments | name, dataset_id, config (JSONB), user_id | -| `training_jobs` | Training tasks | status, model_type, experiment_id | -| `training_results` | Model metrics | accuracy, precision, f1_score, job_id | -| `training_runs` | Run tracking | start_time, end_time, error_message | -| `training_logs` | Progress logs | message, metrics_json, timestamp | -| `prediction_batches` | Predictions | input_file_path, output_file_path, status | - ---- - -## ๐ŸŽ“ Technical Highlights - -### What We Built -- **Full ML Pipeline:** Complete workflow from data upload to predictions -- **8 Production-Ready Models:** 4 classification + 4 regression algorithms (Logistic/Linear Regression, Random Forest, XGBoost, LightGBM) -- **Async Training:** Celery + Redis for non-blocking model training -- **Type-Safe Frontend:** Full TypeScript coverage with proper interfaces -- **Interactive Viz:** Server-side Plotly, client-side Recharts for data exploration -- **Microservices Architecture:** Separate containers for API, worker, database, cache - -### Challenges Overcome -1. **Long-running Tasks:** Implemented Celery task queue to handle training jobs that can take minutes -2. **State Synchronization:** Real-time status updates between training worker and UI via polling -3. **Type Safety:** Consistent TypeScript types across 50+ React components -4. **Data Validation:** Robust Pydantic schemas preventing bad data from reaching ML pipeline -5. **Model Persistence:** Efficient joblib serialization and retrieval of trained models - -### Key Architecture Decisions -| Decision | Rationale | -|----------|-----------| -| FastAPI over Flask | Native async support, automatic OpenAPI docs, better performance | -| Celery for training | Prevents API timeouts on long-running ML tasks | -| React Context API | Simpler than Redux for our scope; sufficient for auth/experiment state | -| Docker Compose | Single-command dev environment with all services | -| PostgreSQL | ACID compliance for experiment/training data | -| shadcn/ui | Accessible components, full customization control | - ---- +NoCodeML is intentionally a coherent ML utility rather than a collection of unrelated AI features. The product should make the workflow easier without hiding what target, features, model, split and metrics were actually used. diff --git a/SUPABASE_HUB_RULES.md b/SUPABASE_HUB_RULES.md new file mode 100644 index 0000000..c9eda89 --- /dev/null +++ b/SUPABASE_HUB_RULES.md @@ -0,0 +1,39 @@ +# NoCodeML โ€” Supabase Project Hub Rules + +This repository uses the shared Supabase **Project Hub** only through the dedicated NoCodeML application scope. + +## Assigned scope + +- App slug: `nocodeml` +- Schema: `nocodeml` +- Repository: `Rishikeshsanin/NoCodeML` + +The slug and schema must match exactly. + +## Before every database change + +1. Read `hub.read_me_first`. +2. Verify the NoCodeML row in `hub.apps`. +3. Confirm its repository is `Rishikeshsanin/NoCodeML` and its schema is `nocodeml`. +4. Run `select hub.assert_app_scope('nocodeml', 'nocodeml');`. +5. Inspect only the `nocodeml` schema and NoCodeML-registered resources. +6. Apply only fully-qualified `nocodeml.*` changes. +7. Verify the result and run Supabase security/performance advisors after meaningful schema or policy work. + +## Forbidden from NoCodeML work + +- Creating application tables in `public`. +- Reading or changing another app schema. +- Cross-app foreign keys or dependencies. +- Unscoped `DROP`, `TRUNCATE`, `DELETE`, or `ALTER` operations. +- Disabling access controls as a shortcut. +- Changing project-wide Auth, keys, region, plan, or shared infrastructure for an app-specific task. +- Exposing database passwords, secret keys, service-role keys, or other server secrets to frontend code. + +## Backend connectivity + +NoCodeML's FastAPI backend may connect to Postgres only with a server-side Project Hub database connection configured so all application SQL resolves to the `nocodeml` schema. Migration tooling must explicitly target the same schema. + +## Frontend connectivity + +The React frontend must never receive privileged Project Hub database credentials. Browser-visible environment variables are treated as public. diff --git a/deploy/free-vm/bootstrap-ubuntu.sh b/deploy/free-vm/bootstrap-ubuntu.sh new file mode 100644 index 0000000..bf71361 --- /dev/null +++ b/deploy/free-vm/bootstrap-ubuntu.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +REPO_URL="https://github.com/Rishikeshsanin/NoCodeML.git" +BRANCH="release/v3-revival" +APP_DIR="${NOCODEML_APP_DIR:-/opt/nocodeml}" + +if [[ "${EUID}" -eq 0 ]]; then + SUDO="" + OWNER="${SUDO_USER:-root}" +else + SUDO="sudo" + OWNER="${USER}" +fi + +printf '\n==> Installing host prerequisites\n' +${SUDO} apt-get update +${SUDO} apt-get install -y ca-certificates curl git docker.io docker-compose-v2 ufw +${SUDO} systemctl enable --now docker + +printf '\n==> Applying host firewall rules\n' +${SUDO} ufw allow OpenSSH +${SUDO} ufw allow 80/tcp +${SUDO} ufw allow 443/tcp +${SUDO} ufw --force enable + +printf '\n==> Preparing NoCodeML checkout\n' +if [[ -d "${APP_DIR}/.git" ]]; then + ${SUDO} git -C "${APP_DIR}" fetch origin "${BRANCH}" + ${SUDO} git -C "${APP_DIR}" checkout "${BRANCH}" + ${SUDO} git -C "${APP_DIR}" pull --ff-only origin "${BRANCH}" +else + ${SUDO} mkdir -p "$(dirname "${APP_DIR}")" + ${SUDO} git clone --branch "${BRANCH}" --single-branch "${REPO_URL}" "${APP_DIR}" +fi + +${SUDO} chown -R "${OWNER}:${OWNER}" "${APP_DIR}" 2>/dev/null || true + +ENV_FILE="${APP_DIR}/Backend/.env.production" +if [[ ! -f "${ENV_FILE}" ]]; then + cp "${APP_DIR}/Backend/.env.production.example" "${ENV_FILE}" + chmod 600 "${ENV_FILE}" +fi + +PUBLIC_IP="$(curl -4fsS --max-time 8 https://api.ipify.org || true)" +if [[ -n "${PUBLIC_IP}" ]]; then + FREE_HOSTNAME="${PUBLIC_IP}.sslip.io" + if grep -q '^PUBLIC_HOSTNAME=REPLACE_WITH_BACKEND_HOSTNAME$' "${ENV_FILE}"; then + sed -i "s#^PUBLIC_HOSTNAME=.*#PUBLIC_HOSTNAME=${FREE_HOSTNAME}#" "${ENV_FILE}" + fi + printf '\nDetected public IPv4: %s\nSuggested free HTTPS hostname: https://%s\n' "${PUBLIC_IP}" "${FREE_HOSTNAME}" +fi + +cat <&2 + exit 1 +fi + +fail_if_placeholder() { + local key="$1" + local value + value="$(grep -E "^${key}=" "${ENV_FILE}" | tail -n1 | cut -d= -f2- || true)" + if [[ -z "${value}" || "${value}" == *REPLACE_* || "${value}" == *"USER:PASSWORD@HOST"* ]]; then + echo "Production value ${key} is missing or still contains a placeholder." >&2 + exit 1 + fi +} + +fail_if_placeholder DATABASE_URL +fail_if_placeholder SECRET_KEY +fail_if_placeholder BACKEND_CORS_ORIGINS +fail_if_placeholder PUBLIC_HOSTNAME + +if grep -Eq '^DB_SCHEMA=(?!nocodeml$)' "${ENV_FILE}" 2>/dev/null; then + echo "DB_SCHEMA must remain exactly 'nocodeml'." >&2 + exit 1 +fi + +cd "${BACKEND_DIR}" + +echo "==> Validating production Compose configuration" +docker compose --env-file .env.production -f docker-compose.production.yaml config >/dev/null + +echo "==> Building NoCodeML V3 containers" +docker compose --env-file .env.production -f docker-compose.production.yaml build --pull + +echo "==> Starting NoCodeML V3" +docker compose --env-file .env.production -f docker-compose.production.yaml up -d + +echo "==> Waiting for API health" +for attempt in $(seq 1 40); do + if docker compose --env-file .env.production -f docker-compose.production.yaml exec -T api \ + python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=4).read()" \ + >/dev/null 2>&1; then + echo "API is healthy." + HOSTNAME_VALUE="$(grep '^PUBLIC_HOSTNAME=' .env.production | cut -d= -f2-)" + echo "Backend URL: https://${HOSTNAME_VALUE}" + echo "API docs: https://${HOSTNAME_VALUE}/docs" + exit 0 + fi + sleep 3 +done + +echo "API did not become healthy in time. Recent logs:" >&2 +docker compose --env-file .env.production -f docker-compose.production.yaml logs --tail=120 api worker redis >&2 +exit 1 diff --git a/deploy/oci/.env.example b/deploy/oci/.env.example new file mode 100644 index 0000000..7243788 --- /dev/null +++ b/deploy/oci/.env.example @@ -0,0 +1,19 @@ +# Public HTTPS hostname served by Caddy. +# sslip.io resolves an embedded public IP automatically, for example: +# nocodeml-api.203.0.113.10.sslip.io +NOCODEML_HOST=nocodeml-api.203.0.113.10.sslip.io + +# Set this to the exact Vercel production origin once the frontend exists. +# During the initial backend-only smoke test you may temporarily use * and then +# immediately replace it with the final https://....vercel.app origin. +FRONTEND_ORIGIN=* + +SESSION_TTL_MINUTES=60 +SESSION_CLEANUP_INTERVAL_SECONDS=300 +SESSION_CLOSE_GRACE_SECONDS=30 +WORKSPACE_TRAINING_WORKERS=1 +WORKSPACE_MAX_MODELS_PER_RUN=8 + +# Optional. Keep secrets only on the server; never commit a real key. +GEMINI_API_KEY= +GEMINI_MODEL=gemini-3.7-flash diff --git a/deploy/oci/Caddyfile b/deploy/oci/Caddyfile new file mode 100644 index 0000000..b21af0e --- /dev/null +++ b/deploy/oci/Caddyfile @@ -0,0 +1,14 @@ +{$NOCODEML_HOST} { + encode zstd gzip + + @health path /health /ready + header @health Cache-Control "no-store" + + reverse_proxy api:8000 + + header { + X-Content-Type-Options "nosniff" + Referrer-Policy "strict-origin-when-cross-origin" + -Server + } +} diff --git a/deploy/oci/bootstrap-ubuntu.sh b/deploy/oci/bootstrap-ubuntu.sh new file mode 100644 index 0000000..ac200c2 --- /dev/null +++ b/deploy/oci/bootstrap-ubuntu.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ "${EUID}" -ne 0 ]]; then + echo "Run with sudo: sudo bash deploy/oci/bootstrap-ubuntu.sh" + exit 1 +fi + +if [[ ! -f deploy/oci/docker-compose.yml ]]; then + echo "Run this script from the NoCodeML repository root." + exit 1 +fi + +ARCH="$(uname -m)" +if [[ "${ARCH}" != "aarch64" && "${ARCH}" != "arm64" ]]; then + echo "Warning: expected Oracle Ampere ARM64, found ${ARCH}. The stack can still run if the ML wheels support this architecture." +fi + +export DEBIAN_FRONTEND=noninteractive +apt-get update +apt-get install -y ca-certificates curl git docker.io docker-compose-v2 +systemctl enable --now docker + +PUBLIC_IP="$(curl -fsS https://api.ipify.org)" +if [[ -z "${PUBLIC_IP}" ]]; then + echo "Could not determine the public IPv4 address." + exit 1 +fi + +HOST="nocodeml-api.${PUBLIC_IP}.sslip.io" +ENV_FILE="deploy/oci/.env" + +if [[ ! -f "${ENV_FILE}" ]]; then + cp deploy/oci/.env.example "${ENV_FILE}" +fi + +python3 - "${ENV_FILE}" "${HOST}" <<'PY' +from pathlib import Path +import sys + +path = Path(sys.argv[1]) +host = sys.argv[2] +lines = path.read_text().splitlines() +out = [] +seen = False +for line in lines: + if line.startswith("NOCODEML_HOST="): + out.append(f"NOCODEML_HOST={host}") + seen = True + else: + out.append(line) +if not seen: + out.append(f"NOCODEML_HOST={host}") +path.write_text("\n".join(out) + "\n") +PY + +# Oracle's VCN/security list must also allow inbound TCP 80 and 443. +# UFW may be disabled on a fresh image; these commands are harmless either way. +ufw allow 22/tcp >/dev/null 2>&1 || true +ufw allow 80/tcp >/dev/null 2>&1 || true +ufw allow 443/tcp >/dev/null 2>&1 || true + +DOCKER_COMPOSE=(docker compose --env-file "${ENV_FILE}" -f deploy/oci/docker-compose.yml) +"${DOCKER_COMPOSE[@]}" up -d --build + +for attempt in $(seq 1 60); do + if curl -fsS "https://${HOST}/health" >/dev/null 2>&1; then + echo + echo "NoCodeML backend is live: https://${HOST}" + echo "Readiness: https://${HOST}/ready" + echo + echo "Next: create/import the Vercel frontend, set VITE_API_URL=https://${HOST}," + echo "then replace FRONTEND_ORIGIN=* in ${ENV_FILE} with the exact Vercel origin" + echo "and run: docker compose --env-file ${ENV_FILE} -f deploy/oci/docker-compose.yml up -d" + exit 0 + fi + sleep 5 +done + +echo "Backend did not become healthy in time. Recent logs:" +"${DOCKER_COMPOSE[@]}" logs --tail=120 +exit 1 diff --git a/deploy/oci/docker-compose.yml b/deploy/oci/docker-compose.yml new file mode 100644 index 0000000..da15ceb --- /dev/null +++ b/deploy/oci/docker-compose.yml @@ -0,0 +1,53 @@ +name: nocodeml-production + +services: + api: + build: + context: ../../Backend + restart: unless-stopped + environment: + ENVIRONMENT: production + SESSION_ROOT_DIR: /tmp/nocodeml-sessions + SESSION_TTL_MINUTES: ${SESSION_TTL_MINUTES:-60} + SESSION_CLEANUP_INTERVAL_SECONDS: ${SESSION_CLEANUP_INTERVAL_SECONDS:-300} + SESSION_CLOSE_GRACE_SECONDS: ${SESSION_CLOSE_GRACE_SECONDS:-30} + WORKSPACE_TRAINING_WORKERS: ${WORKSPACE_TRAINING_WORKERS:-1} + WORKSPACE_MAX_MODELS_PER_RUN: ${WORKSPACE_MAX_MODELS_PER_RUN:-8} + BACKEND_CORS_ORIGINS: ${FRONTEND_ORIGIN} + GEMINI_API_KEY: ${GEMINI_API_KEY:-} + GEMINI_MODEL: ${GEMINI_MODEL:-gemini-3.7-flash} + tmpfs: + - /tmp/nocodeml-sessions:rw,noexec,nosuid,size=6g,mode=0700,uid=10001,gid=10001 + - /tmp/nocodeml-legacy:rw,noexec,nosuid,size=256m,mode=0700,uid=10001,gid=10001 + - /tmp/nocodeml-artifacts:rw,noexec,nosuid,size=512m,mode=0700,uid=10001,gid=10001 + expose: + - "8000" + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=3)"] + interval: 30s + timeout: 5s + start_period: 30s + retries: 3 + + caddy: + image: caddy:2-alpine + restart: unless-stopped + depends_on: + api: + condition: service_healthy + environment: + NOCODEML_HOST: ${NOCODEML_HOST} + ports: + - "80:80" + - "443:443" + volumes: + - ./Caddyfile:/etc/caddy/Caddyfile:ro + - caddy_data:/data + - caddy_config:/config + +volumes: + caddy_data: + caddy_config: + +# Only Caddy certificate/config state is persistent. Visitor ML workspaces are +# tmpfs and are therefore lost on container/VM restart in addition to TTL cleanup. diff --git a/vercel.json b/vercel.json new file mode 100644 index 0000000..7e9a2a7 --- /dev/null +++ b/vercel.json @@ -0,0 +1,24 @@ +{ + "$schema": "https://openapi.vercel.sh/vercel.json", + "framework": "vite", + "installCommand": "npm --prefix Frontend ci", + "buildCommand": "npm --prefix Frontend run build", + "outputDirectory": "Frontend/dist", + "rewrites": [ + { + "source": "/(.*)", + "destination": "/index.html" + } + ], + "headers": [ + { + "source": "/(.*)", + "headers": [ + { "key": "X-Content-Type-Options", "value": "nosniff" }, + { "key": "X-Frame-Options", "value": "DENY" }, + { "key": "Referrer-Policy", "value": "strict-origin-when-cross-origin" }, + { "key": "Permissions-Policy", "value": "camera=(), microphone=(), geolocation=()" } + ] + } + ] +}