From cd80a80feb33f50bbebc30a3f1d95b5aec8a57d2 Mon Sep 17 00:00:00 2001 From: Paul Castillo Date: Thu, 21 May 2026 07:46:13 +0000 Subject: [PATCH 1/3] implementation of absolute path for inputs and outputs in Docker + read me update --- sen2vm-notebook/README_Notebooks.md | 52 +++++++++-- sen2vm-notebook/src/notebook.ipynb | 136 ++++++++++++++++------------ 2 files changed, 122 insertions(+), 66 deletions(-) diff --git a/sen2vm-notebook/README_Notebooks.md b/sen2vm-notebook/README_Notebooks.md index 78c56020..652dfa57 100644 --- a/sen2vm-notebook/README_Notebooks.md +++ b/sen2vm-notebook/README_Notebooks.md @@ -32,7 +32,7 @@ Actually, this notebook can generate an inverse grid but can not use it to apply * Required data: * **Sentinel-2 L1B product** : /DATASTRIP and /GRANULE. - * **DEM files** placed in the appropriate directory + * **DEM files** ## Mandatory Directory Structure @@ -40,16 +40,48 @@ Actually, this notebook can generate an inverse grid but can not use it to apply WORKDIR │ ├── DATA/ -│ ├── DEM/ # Put your DEM files here -| ├── GEOID/ # Put your GEOID files here (Optional) -│ └── / # Place the full L1B product here -| ├── DATASTRIP # Required -| ├── GRANULE # Required -| └── ... +│ └── GEOID/ # Put your GEOID files here (Optional) └── ... ``` -If the GEOID folder is empty, the notebopok will automaticaly use the geoid provided by sen2vm-core +## Inputs + +Several inputs are needed : +-L1B S2 product under EUP.SAFE format : +```bash + / + ├── DATASTRIP # Required + ├── GRANULE # Required + ├── S2*OPER_MTD_SAFL1B_PDMC*.xml #Required + └── ... +``` +-Digital Elevation Model (DEM) +-GIPP (if the path given in the notebook cell n°1 is empty (i.e ""), GIPP can be donwloaded and cloned from the GIT in /WORKSPACE/DATA + +Thanks to recent updates, input data can now be handled in two different ways: + +Using absolute paths (recommended) : +You can directly provide full paths to your data (L1B, DEM, GIPP), wherever they are located on your system. +This avoids copying large datasets and allows more flexibility. + +Using the default WORKDIR/DATA structure : +Alternatively, you can place all inputs inside the WORKDIR/DATA directory. + +## Directory Structure after execution +```bash +WORKDIR +│ +├── DATA/ +│ ├── bulletin*.txt # IERS prediction of earth exploration (downloaded by the cell n°3) +│ └── GEOID +│ +├── output/ # output after orthorectification and mosaic (.tif) +│ ├── GDAL_OUTPUT_ORTHO +│ └── GDAL_OUTPUT_MOSAIC +│ +├── src/ # contains a .sh used to run in the gdal docker generated by the notebook +└── UserConf # contains .txt config files generated by the notebook +``` ## Python Environment Setup @@ -71,8 +103,10 @@ In the first cell of the notebook: * The working directory * The L1B product + * The GIPP directory * The output directory - + * The DEM directory + 2. Adjust configuration parameters for: * Sen2VM diff --git a/sen2vm-notebook/src/notebook.ipynb b/sen2vm-notebook/src/notebook.ipynb index 8cd94e73..141b495a 100644 --- a/sen2vm-notebook/src/notebook.ipynb +++ b/sen2vm-notebook/src/notebook.ipynb @@ -2,16 +2,16 @@ "cells": [ { "cell_type": "markdown", - "id": "e8bbfd73", + "id": "8e5ed8d8-c718-4f80-a220-464bbc67c748", "metadata": {}, "source": [ - "# === USER CONFIGURATION ===\n" + "# === USER CONFIGURATION ===" ] }, { "cell_type": "code", - "execution_count": null, - "id": "3be1da11", + "execution_count": 1, + "id": "2b816cd9-5254-452b-bced-2943eb6b7aec", "metadata": {}, "outputs": [], "source": [ @@ -26,14 +26,21 @@ "# /!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\\n", "\n", "# Path to downloaded product\n", - "PATH_L1B_DATA = \"PATH/TO/L1B/PRODUCT\"\n", + "PATH_L1B_DATA = \"PATH/TO/L1B_PRODUCT\"\n", + "\n", + "# Path to GIPP directory\n", + "PATH_GIPP = \"PATH/TO/GIPP\"\n", "# /!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\\n", - "# /!\\/!\\/!\\ CAREFUL, for now PATH_L1B_DATA shall be inside WORKID/DATA and respect the one described in README_Notebooks.md\n", + "# /!\\/!\\/!\\ If the \"PATH_GIPP is empty (i.e = \"\"), you may execute the cell \"# GIPP database download (Optional)\", so the gipp can be cloned \n", + "# cloned from git, they will be put in /WORKDIR/DATA/sen2vm-gipp-database\n", + "# Please note that this current notebook will search for a subfolder with mission S2[A/B/C] inside the GIPP folder\n", "# /!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\\n", "\n", - "# Output folder chosen by the user\n", - "INVERSE_OUTPUT_FOLDER = \"PATH/TO/OUTPUT/FOLDER\" # Used only if GRID_MODE = \"inverse\"\n", + "# Path to DEM directory\n", + "PATH_DEM = \"PATH/TO/DEM\"\n", "\n", + "# Output folder chosen by the user\n", + "INVERSE_OUTPUT_FOLDER = \"PATH/TO/OUTPUT_FOLDER\" # where the exits are donwloaded\n", "\n", "# === SEN2VM OPTIONS ===\n", "\n", @@ -42,12 +49,12 @@ "# /!\\/!\\/!\\ CAREFUL, for now only direct shall be used with a full product (no missing granules)\n", "# /!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\\n", "\n", - "UTM_EPSG = 32636 # UTM zone EPSG code for the ROI\n", + "UTM_EPSG = 32628 # UTM zone EPSG code for the ROI\n", "LOCATION = {\n", - " \"ul_x\": 210000, \n", - " \"ul_y\": 3504541, \n", - " \"lr_x\": 436397, \n", - " \"lr_y\": 3296817 \n", + " \"ul_x\": 281719,\n", + " \"ul_y\": 3640370,\n", + " \"lr_x\": 336222,\n", + " \"lr_y\": 3611287\n", "}\n", "\n", "# Grid step in pixels for sen2vm grid generation \n", @@ -59,14 +66,12 @@ " \"60m_bands\": 1 # Grid step: ~DEM90/2 = 45m/60m ≈ 1 pixel (one point every 1 pixel at 60m, or 2 is also fine)\n", "}\n", "\n", - "\n", "# === GDAL ORTHO OPTIONS ===\n", "\n", "ORTHO_SETTINGS = {\n", - " \"keep_bands\": [\"B03\"], # list of bands to keep for orthorectification\n", - " \"keep_detectors\": [\"11\",\"10\"] # list of the detectors to keep \n", - "}\n", - " \n" + " \"keep_bands\": [\"B02\"], # list of bands to keep for orthorectification\n", + " \"keep_detectors\": [\"01\",\"02\",\"03\",\"04\",\"05\",\"06\",\"07\",\"08\",\"09\",\"10\",\"11\",\"12\"] # list of the detectors to keep \n", + "}\n" ] }, { @@ -86,6 +91,7 @@ "source": [ "# === GIPPs ===\n", "# This step is optional if the Database was already downloaded or if you want to use your own GIPP\n", + "# If the PATH_GIPP isn't equal to \"\", this cell won't work, if PATH_GIPP= \"\", this cell will clone the git GIPP folder inside WORKDIR/DATA\n", "# Please note that this current notebook will search for a subfolder with mission S2[A/B/C] inside the GIPP folder\n", "\n", "import os\n", @@ -93,7 +99,11 @@ "import tarfile\n", "import re\n", "from datetime import datetime\n", + "import sys\n", "\n", + "if len(PATH_GIPP) != 0:\n", + " print(\"Stopping because PATH_GIPP is not empty\")\n", + " sys.exit(0)\n", "\n", "gipp_dir = os.path.join(WORKDIR, \"DATA\") \n", "os.makedirs(gipp_dir, exist_ok=True) \n", @@ -103,21 +113,20 @@ "# =====================================================================\n", "\n", "gipp_repo_name = \"sen2vm-gipp-database\"\n", - "gipp_repo_path = os.path.join(gipp_dir, gipp_repo_name)\n", + "PATH_GIPP = os.path.join(gipp_dir, gipp_repo_name)\n", "\n", "# Delete repository if exists\n", - "if os.path.exists(gipp_repo_path):\n", - " print(f\"Removing existing repository: {gipp_repo_path}\")\n", - " shutil.rmtree(gipp_repo_path)\n", + "if os.path.exists(PATH_GIPP):\n", + " print(f\"Removing existing repository: {PATH_GIPP}\")\n", + " shutil.rmtree(PATH_GIPP)\n", "\n", "# Clone repository fresh\n", "print(\"Cloning sen2vm-gipp-database...\")\n", - "!git clone https://github.com/sen2vm/sen2vm-gipp-database.git {gipp_repo_path}\n", + "!git clone https://github.com/sen2vm/sen2vm-gipp-database.git {PATH_GIPP}\n", "print(\"Clone complete.\\n\")\n", "\n", - "\n", - "\n", - "print(\"GIPP processing finished successfully.\")\n" + "print(\"GIPP processing finished successfully.\")\n", + "print(\"GIPP donwloaded in :\", PATH_GIPP)\n" ] }, { @@ -310,11 +319,9 @@ "# 2. Docker paths inside /workspace\n", "# =====================================================\n", "\n", - "safe_name = os.path.basename(PATH_L1B_DATA)\n", - "\n", - "docker_l1b = f\"/workspace/DATA/{safe_name}\"\n", - "docker_dem = \"/workspace/DATA/DEM\"\n", - "docker_gipp = f\"/workspace/DATA/sen2vm-gipp-database/{mission}\"\n", + "docker_l1b = \"/data/L1B\"\n", + "docker_dem = \"/data/DEM\"\n", + "docker_gipp = f\"/data/GIPP/{mission}\"\n", "\n", "# =====================================================\n", "# 3. Geoid management\n", @@ -552,6 +559,9 @@ "cmd_run = [\n", " \"docker\", \"run\",\n", " \"--rm\",\n", + " \"-v\", f\"{PATH_L1B_DATA}:/data/L1B\",\n", + " \"-v\", f\"{PATH_DEM}:/data/DEM\",\n", + " \"-v\", f\"{PATH_GIPP}:/data/GIPP\",\n", " \"-v\", f\"{WORKDIR}:/workspace\", \n", " \"sen2vm\",\n", " \"-c\", config_inside,\n", @@ -560,20 +570,17 @@ "\n", "print(\"Running Docker container...\\n\")\n", "print(\"Command:\", \" \".join(cmd_run), \"\\n\")\n", - "\n", "subprocess.run(cmd_run, check=True)\n", - "\n", "print(\"\\nDocker execution complete.\\n\")\n", "\n", "# =====================================================\n", "# 3. REMOVE DOCKER IMAGE\n", "# =====================================================\n", - "\n", + "\"\"\"\n", "print(\"Removing Docker image 'sen2vm'...\")\n", - "\n", "subprocess.run([\"docker\", \"rmi\", \"-f\", \"sen2vm\"], check=True)\n", - "\n", - "print(\"Docker image removed.\\n\")" + "print(\"Docker image removed.\\n\")\n", + "\"\"\"\n" ] }, { @@ -581,7 +588,7 @@ "id": "e54044d2", "metadata": {}, "source": [ - "# Generate Orthorectification images" + " # Generate Orthorectification images" ] }, { @@ -598,6 +605,14 @@ "import re\n", "\n", "# =====================================================\n", + "# SAFETY CHECKS\n", + "# =====================================================\n", + "assert os.path.exists(PATH_L1B_DATA), \"L1B path missing\"\n", + "\n", + "os.makedirs(INVERSE_OUTPUT_FOLDER, exist_ok=True)\n", + "assert os.path.exists(INVERSE_OUTPUT_FOLDER), \"Output folder missing\"\n", + "\n", + "# =====================================================\n", "# Locate product name\n", "# =====================================================\n", "product = os.path.basename(os.path.normpath(PATH_L1B_DATA))\n", @@ -678,10 +693,10 @@ "# =====================================================\n", "# Output directories\n", "# =====================================================\n", - "OUTDIR = os.path.join(WORKDIR, \"DATA\", \"GDAL_OUTPUT_ORTHO\")\n", - "os.makedirs(OUTDIR, exist_ok=True)\n", + "os.makedirs(os.path.join(INVERSE_OUTPUT_FOLDER, \"GDAL_OUTPUT_ORTHO\"), exist_ok=True)\n", + "os.makedirs(os.path.join(INVERSE_OUTPUT_FOLDER, \"GDAL_OUTPUT_MOSAIC\"), exist_ok=True)\n", "\n", - "OUTDIR_DOCKER = \"/workspace/DATA/GDAL_OUTPUT_ORTHO\"\n", + "print(\"Output folder:\", INVERSE_OUTPUT_FOLDER)\n", "\n", "# =====================================================\n", "# Build GDAL docker\n", @@ -733,10 +748,10 @@ " f.write(f\"\"\"#!/bin/bash\n", "set +e\n", "\n", - "cd /workspace/DATA/{product}\n", + "cd /data/L1B\n", "\n", - "OUT_ORTHO=\"/workspace/DATA/GDAL_OUTPUT_ORTHO\"\n", - "OUT_MOSAIC=\"/workspace/DATA/GDAL_OUTPUT_MOSAIC\"\n", + "OUT_ORTHO=\"/output/GDAL_OUTPUT_ORTHO\"\n", + "OUT_MOSAIC=\"/output/GDAL_OUTPUT_MOSAIC\"\n", "\n", "mkdir -p \"$OUT_ORTHO\"\n", "mkdir -p \"$OUT_MOSAIC\"\n", @@ -771,15 +786,14 @@ "echo \"=== ORTHORECTIFICATION ===\"\n", "\n", "for VRT in {vrt_array}; do\n", - " BASENAME=\"$VRT\"\n", - " OUT=\"$OUT_ORTHO/${{BASENAME}}_ortho.tif\"\n", - "\n", + " OUT=\"$OUT_ORTHO/${{VRT}}_ortho.tif\"\n", + " \n", " echo \"----------------------------------------\"\n", " echo \"Processing VRT: $VRT\"\n", " echo \"----------------------------------------\"\n", "\n", " rm -f \"$OUT\"\n", - "\n", + " \n", " # Get resolution for this band (Issue #61)\n", " RESOLUTION=$(get_band_resolution \"$VRT\")\n", " echo \"Band resolution: $RESOLUTION m\"\n", @@ -796,8 +810,6 @@ " -co COMPRESS=LZW \\\\\n", " -co TILED=YES \\\\\n", " -overwrite\n", - "\n", - " echo \"\"\n", "done\n", "\n", "echo \"\"\n", @@ -808,16 +820,15 @@ " echo \"----------------------------------------\"\n", " echo \"Creating mosaic for band: $BAND\"\n", " echo \"----------------------------------------\"\n", - "\n", - " INPUT_FILES=($(ls $OUT_ORTHO/*_${{BAND}}_ortho.tif 2>/dev/null))\n", + " \n", + " INPUT_FILES=($OUT_ORTHO/*_${{BAND}}_ortho.tif)\n", "\n", " if [ ${{#INPUT_FILES[@]}} -eq 0 ]; then\n", " echo \"No ortho images found for band $BAND\"\n", " continue\n", " fi\n", - "\n", " OUTPUT=\"$OUT_MOSAIC/ORTHO_mosaic_${{BAND}}.tif\"\n", - "\n", + " \n", " # Use gdal_merge.py instead of gdalwarp to avoid double resampling \n", " # All ortho images should have the same resolution and geometry\n", " # No resampling needed, just assembly\n", @@ -839,7 +850,6 @@ "\"\"\")\n", "\n", "os.chmod(gdal_script_path, 0o755)\n", - "print(\"Generated:\", gdal_script_path)\n", "\n", "# =====================================================\n", "# Run GDAL docker\n", @@ -849,7 +859,9 @@ "cmd_run = [\n", " \"docker\", \"run\",\n", " \"--rm\",\n", + " \"-v\", f\"{PATH_L1B_DATA}:/data/L1B\",\n", " \"-v\", f\"{WORKDIR}:/workspace\",\n", + " \"-v\", f\"{INVERSE_OUTPUT_FOLDER}:/output\",\n", " \"gdal-latest\",\n", " \"/workspace/src/gdal_ortho.sh\"\n", "]\n", @@ -858,13 +870,23 @@ "subprocess.run(cmd_run, check=True)\n", "print(\"\\nGDAL ortho + mosaic complete.\\n\")\n", "\n", + "\"\"\"\n", "# =====================================================\n", "# Cleanup docker image\n", "# =====================================================\n", "print(\"Removing gdal-latest image...\\n\")\n", "subprocess.run([\"docker\", \"rmi\", \"-f\", \"gdal-latest\"], check=True)\n", - "print(\"GDAL image removed.\\n\")\n" + "print(\"GDAL image removed.\\n\")\n", + "\"\"\"" ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "257b2ddd-8b9d-4c21-9233-0223c2a8b7d0", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { @@ -883,7 +905,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.11" + "version": "3.13.11" } }, "nbformat": 4, From eecd79b9d43a2dbff30629f36a5c5f6077c3a40f Mon Sep 17 00:00:00 2001 From: Paul Castillo Date: Fri, 22 May 2026 11:39:06 +0000 Subject: [PATCH 2/3] Apply review comments : update of the README, separate notebook for static inputs --- sen2vm-notebook/README_Notebooks.md | 80 ++++--- .../src/inputs-download-notebook.ipynb | 152 +++++++++++++ sen2vm-notebook/src/notebook.ipynb | 212 +++++++----------- 3 files changed, 284 insertions(+), 160 deletions(-) create mode 100644 sen2vm-notebook/src/inputs-download-notebook.ipynb diff --git a/sen2vm-notebook/README_Notebooks.md b/sen2vm-notebook/README_Notebooks.md index 652dfa57..31e53631 100644 --- a/sen2vm-notebook/README_Notebooks.md +++ b/sen2vm-notebook/README_Notebooks.md @@ -16,6 +16,15 @@ # Sen2VM Notebook Processing Workflow This repository provides a complete workflow to run **Sen2VM** inside Docker and generate orthorectified and mosaicked outputs from Sentinel-2 L1B data. + +Two notebooks are provided: + +* **inputs-download-notebook.ipynb** + Downloads all required input data (GEOID and GIPP) from the Git repository. + +* **notebook.ipynb** + Runs Sen2VM using the downloaded inputs, along with the IERS file and L1B product, to produce orthorectified and mosaicked images. + The project is designed and tested on **Linux**. It may not work reliably on **Windows**. Actually, this notebook can generate an inverse grid but can not use it to apply the orthorectification. @@ -36,18 +45,12 @@ Actually, this notebook can generate an inverse grid but can not use it to apply ## Mandatory Directory Structure -```bash -WORKDIR -│ -├── DATA/ -│ └── GEOID/ # Put your GEOID files here (Optional) -└── ... -``` +Only a `WORKDIR` folder is required. All outputs, as well as intermediate files generated during execution, will be stored there. ## Inputs Several inputs are needed : --L1B S2 product under EUP.SAFE format : +* L1B S2 product under EUP.SAFE format : ```bash / ├── DATASTRIP # Required @@ -55,25 +58,21 @@ Several inputs are needed : ├── S2*OPER_MTD_SAFL1B_PDMC*.xml #Required └── ... ``` --Digital Elevation Model (DEM) --GIPP (if the path given in the notebook cell n°1 is empty (i.e ""), GIPP can be donwloaded and cloned from the GIT in /WORKSPACE/DATA +* Digital Elevation Model (DEM) + +* IERS (prediction of Earth orientation, IERS can be donwloaded using **notebook.ipynb** (cell number 2)) -Thanks to recent updates, input data can now be handled in two different ways: +* GIPP (GIPP can be downloaded from the GIT repositery using **inputs-download-notebook.ipynb**) -Using absolute paths (recommended) : -You can directly provide full paths to your data (L1B, DEM, GIPP), wherever they are located on your system. -This avoids copying large datasets and allows more flexibility. +* GEOID (GEOID can also be downloaded from the GIT repositery using **inputs-download-notebook.ipynb** ) -Using the default WORKDIR/DATA structure : -Alternatively, you can place all inputs inside the WORKDIR/DATA directory. +Thanks to a recent update (#issue 56), now the differents inputs path are absolute path. ## Directory Structure after execution ```bash WORKDIR │ -├── DATA/ -│ ├── bulletin*.txt # IERS prediction of earth exploration (downloaded by the cell n°3) -│ └── GEOID +├── bulletin*.txt # IERS prediction of earth exploration (downloaded by the cell n°3) │ ├── output/ # output after orthorectification and mosaic (.tif) │ ├── GDAL_OUTPUT_ORTHO @@ -95,39 +94,58 @@ pip install -r requirements.txt Select the virtual environment kernel in your Jupyter session. -## Notebook Configuration +## Notebooks Configuration -In the first cell of the notebook: +In **inputs-download-notebook.ipynb** : +In the first cell of the notebook : -1. Set the path to: +1. Set the paths to: + + * The directory where you want the GIPP files to be downloaded + * The directory where you want the GEOID files to be downloaded + +In **notebook.ipynb** : +In the first cell of the notebook : + +1. Set the absolute paths to: * The working directory * The L1B product * The GIPP directory - * The output directory + * The GEOID directory * The DEM directory + * The output directory + * The IERS file (if you do not have the IERS file, put `""`, and one will be automatically ed to the working directory) 2. Adjust configuration parameters for: * Sen2VM * Orthorectification settings + +3. Specify whether the docker images should be removed ## Processing Steps -Execute the notebook cell by cell in the following order: +If you do not have your own GIPP and GEOID files, or if they are not already downloaded, +execute the **inputs-download-notebook.ipynb** cell by cell in the following order : -1. Variable definitions +1. Path definitions 2. Clone `sen2vm-gipp`, and manage GIPP assets -3. Automatic download of the IERS bulletin -4. Generation of `config.json` in: `/WORKDIR/UserConf` -5. Generation of `params.json` in: `/WORKDIR/UserConf` -6. Execution of sen2vm inside Docker -7. Generation of a `.sh` script, then execution inside a second Docker container running the latest GDAL: +3. Copy `Geoid` folder from `"*/sen2vm-core/src/test/resources/DEM_GEOID"` + +Then, execute the **notebook.ipynb** cell by cell : + +1. Variable and path definitions +2. Automatic download of the IERS bulletin if none is provided by the user +3. Generation of `config.json` in: `/WORKDIR/UserConf` +4. Generation of `params.json` in: `/WORKDIR/UserConf` +5. Execution of sen2vm inside Docker +6. Generation of a `.sh` script, then execution inside a second Docker container running the latest GDAL: * Orthorectification by band * Mosaicking -Docker images are automatically cleaned up after each execution. +Docker images can be removed or kept depending on the value of `REMOVE_DOCKER_IMAGE` in step 1. ## Execution diff --git a/sen2vm-notebook/src/inputs-download-notebook.ipynb b/sen2vm-notebook/src/inputs-download-notebook.ipynb new file mode 100644 index 00000000..48b6f870 --- /dev/null +++ b/sen2vm-notebook/src/inputs-download-notebook.ipynb @@ -0,0 +1,152 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "e71d30a3-b1dc-48f9-b7a0-1ee82f8cc775", + "metadata": {}, + "source": [ + "# === USER CONFIGURATION ===" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "0d377684-c4b1-4faa-a16f-02d7c8a58105", + "metadata": {}, + "outputs": [], + "source": [ + "#In this cell, you may indicate where you want the inputs to be donwloaded\n", + "\n", + "# Path where to put the GIPP directory\n", + "PATH_TO_GIPP = \"PATH/WHERE/TO/PUT/GIPP\"\n", + "\n", + "# Path where to put the GEOID directory\n", + "PATH_TO_GEOID = \"PATH/WHERE/TO/PUT/GEOID\"" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "93bc15d2-9c05-4cc2-82e5-3533e1bbcfd0", + "metadata": {}, + "outputs": [], + "source": [ + "# GIPP database download" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "52f77292-b1d3-42cd-817f-24b9bea7e520", + "metadata": {}, + "outputs": [], + "source": [ + "# === GIPPs ===\n", + "# This step is optional if the Database was already downloaded or if you want to use your own GIPP,\n", + "# It will donwload from GITHUB the directory sen2vm-gipp-database at PATH_GIPP\n", + "\n", + "import os\n", + "import shutil\n", + "import tarfile\n", + "import re\n", + "from datetime import datetime\n", + "import sys\n", + "\n", + "if not os.path.exists(PATH_TO_GIPP):\n", + " print(\"The folder where you want to download the GIPP is missing\")\n", + " sys.exit() \n", + "\n", + "# =====================================================================\n", + "# 1) CLONE sen2vm-gipp-database\n", + "# =====================================================================\n", + "\n", + "gipp_repo_name = \"sen2vm-gipp-database\"\n", + "PATH_GIPP = os.path.join(PATH_TO_GIPP, gipp_repo_name)\n", + "\n", + "# Delete repository if exists\n", + "if os.path.exists(PATH_GIPP):\n", + " print(f\"Removing existing repository: {PATH_GIPP}\")\n", + " shutil.rmtree(PATH_GIPP)\n", + "\n", + "# Clone repository fresh\n", + "print(\"Cloning sen2vm-gipp-database...\")\n", + "!git clone https://github.com/sen2vm/sen2vm-gipp-database.git {PATH_GIPP}\n", + "print(\"Clone complete.\\n\")\n", + "\n", + "print(\"GIPP processing finished successfully.\")\n", + "print(\"GIPP donwloaded in :\", PATH_GIPP)\n" + ] + }, + { + "cell_type": "markdown", + "id": "0e8b2d6d-5e67-4145-aa04-42d99671d36b", + "metadata": {}, + "source": [ + "# GEOID folder copy from github" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c781bbdc-3aa7-48d1-9284-5f5c507ed265", + "metadata": {}, + "outputs": [], + "source": [ + "# === GEOID ===\n", + "# This step is optional if the GEOID are already copied, or if you want to you use your own GEOID,\n", + "# It will copy the folder GEOID from \"*/sen2vm-core/src/test/resources/DEM_GEOID\" to PATH_TO_GEOID\n", + "\n", + "import os\n", + "import sys\n", + "import shutil\n", + "\n", + "if not os.path.exists(PATH_TO_GEOID):\n", + " print(\"The folder where you want to download the GEOID is missing\")\n", + " sys.exit() \n", + "\n", + "GEOID_DIR = os.path.join(PATH_TO_GEOID, \"GEOID\")\n", + "\n", + "if os.path.exists(GEOID_DIR):\n", + " shutil.rmtree(GEOID_DIR)\n", + "\n", + "os.makedirs(GEOID_DIR)\n", + "\n", + "# Notebook location (NOT relative to CWD)\n", + "notebook_dir = os.getcwd()\n", + "\n", + "# Relative path to DEM_GEOID from notebook\n", + "internal_geoid_dir = os.path.abspath(os.path.join(\n", + " notebook_dir,\n", + " \"..\", \"..\", \"src\", \"test\", \"resources\", \"DEM_GEOID\"\n", + "))\n", + "\n", + "for f in os.listdir(internal_geoid_dir):\n", + " src = os.path.join(internal_geoid_dir, f)\n", + " dst = os.path.join(GEOID_DIR, f)\n", + " shutil.copy(src, dst)\n", + "print(\"Done copying\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.11" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/sen2vm-notebook/src/notebook.ipynb b/sen2vm-notebook/src/notebook.ipynb index 141b495a..4ef98bce 100644 --- a/sen2vm-notebook/src/notebook.ipynb +++ b/sen2vm-notebook/src/notebook.ipynb @@ -21,9 +21,6 @@ "# /!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\\n", "\n", "WORKDIR = \"PATH/TO/WORKDIR\" # Working directory for sen2vm processing\n", - "# /!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\\n", - "# /!\\/!\\/!\\ CAREFUL, the structure of the WORKDIR shall respect the one described in README_Notebooks.md\n", - "# /!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\\n", "\n", "# Path to downloaded product\n", "PATH_L1B_DATA = \"PATH/TO/L1B_PRODUCT\"\n", @@ -31,16 +28,31 @@ "# Path to GIPP directory\n", "PATH_GIPP = \"PATH/TO/GIPP\"\n", "# /!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\\n", - "# /!\\/!\\/!\\ If the \"PATH_GIPP is empty (i.e = \"\"), you may execute the cell \"# GIPP database download (Optional)\", so the gipp can be cloned \n", - "# cloned from git, they will be put in /WORKDIR/DATA/sen2vm-gipp-database\n", - "# Please note that this current notebook will search for a subfolder with mission S2[A/B/C] inside the GIPP folder\n", + "# /!\\/!\\/!\\ If you do not have your own GIPP folder, you may use the inputs-download-notebook to download it, then you may indicate the path were you \n", + "# /!\\/!\\/!\\ downloaded it here in PATH_GIPP\n", + "# /!\\/!\\/!\\ If you have your own GIPP folder, please note that this current notebook will search for a subfolder with mission S2[A/B/C] inside the GIPP folder\n", "# /!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\\n", "\n", - "# Path to DEM directory\n", + "# Path to DEM directory \n", "PATH_DEM = \"PATH/TO/DEM\"\n", "\n", + "# Path to GEOID directory\n", + "PATH_GEOID = \"PATH/TO/GEOID\"\n", + "# /!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\\n", + "# /!\\/!\\/!\\ If you do not have your own GEOID folder, you may use the inputs-download-notebook to download it, then you may indicate the path were you\n", + "# /!\\/!\\/!\\ downloaded it here in \n", + "# /!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\\n", + "\n", + "# Path to IERS file\n", + "PATH_IERS = \"PATH/TO/IERS_FILE\"\n", + "# /!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\\n", + "# /!\\/!\\/!\\ If you do not have your own IERS, you may let this path empy (i.e. \"\") and execute the cell number 2 named #IERS Download\n", + "# /!\\/!\\/!\\ In this case, the IERS will be downloaded in the WORKDIR\n", + "# /!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\\n", + "\n", "# Output folder chosen by the user\n", - "INVERSE_OUTPUT_FOLDER = \"PATH/TO/OUTPUT_FOLDER\" # where the exits are donwloaded\n", + "OUTPUT_FOLDER = \"PATH/TO/OUTPUT_FOLDER\" # where the exits are donwloaded\n", + "\n", "\n", "# === SEN2VM OPTIONS ===\n", "\n", @@ -71,62 +83,21 @@ "ORTHO_SETTINGS = {\n", " \"keep_bands\": [\"B02\"], # list of bands to keep for orthorectification\n", " \"keep_detectors\": [\"01\",\"02\",\"03\",\"04\",\"05\",\"06\",\"07\",\"08\",\"09\",\"10\",\"11\",\"12\"] # list of the detectors to keep \n", - "}\n" - ] - }, - { - "cell_type": "markdown", - "id": "c9e95184", - "metadata": {}, - "source": [ - "# GIPP database download (Optional)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a4e410c1", - "metadata": {}, - "outputs": [], - "source": [ - "# === GIPPs ===\n", - "# This step is optional if the Database was already downloaded or if you want to use your own GIPP\n", - "# If the PATH_GIPP isn't equal to \"\", this cell won't work, if PATH_GIPP= \"\", this cell will clone the git GIPP folder inside WORKDIR/DATA\n", - "# Please note that this current notebook will search for a subfolder with mission S2[A/B/C] inside the GIPP folder\n", - "\n", - "import os\n", - "import shutil\n", - "import tarfile\n", - "import re\n", - "from datetime import datetime\n", - "import sys\n", - "\n", - "if len(PATH_GIPP) != 0:\n", - " print(\"Stopping because PATH_GIPP is not empty\")\n", - " sys.exit(0)\n", - "\n", - "gipp_dir = os.path.join(WORKDIR, \"DATA\") \n", - "os.makedirs(gipp_dir, exist_ok=True) \n", - "\n", - "# =====================================================================\n", - "# 1) CLONE sen2vm-gipp-database\n", - "# =====================================================================\n", - "\n", - "gipp_repo_name = \"sen2vm-gipp-database\"\n", - "PATH_GIPP = os.path.join(gipp_dir, gipp_repo_name)\n", - "\n", - "# Delete repository if exists\n", - "if os.path.exists(PATH_GIPP):\n", - " print(f\"Removing existing repository: {PATH_GIPP}\")\n", - " shutil.rmtree(PATH_GIPP)\n", + "}\n", "\n", - "# Clone repository fresh\n", - "print(\"Cloning sen2vm-gipp-database...\")\n", - "!git clone https://github.com/sen2vm/sen2vm-gipp-database.git {PATH_GIPP}\n", - "print(\"Clone complete.\\n\")\n", "\n", - "print(\"GIPP processing finished successfully.\")\n", - "print(\"GIPP donwloaded in :\", PATH_GIPP)\n" + "# === Docker Options ===\n", + "# Building Docker images may take several minutes.\n", + "# By setting REMOVE_DOCKER_IMAGE = False, the images are kept,\n", + "# which significantly speeds up subsequent executions.\n", + "#\n", + "# However, Docker images use disk space (~4 GB in this case).\n", + "#\n", + "# Docker images can be manually removed using:\n", + "# docker images\n", + "# docker rmi \n", + "#\n", + "REMOVE_DOCKER_IMAGE = False # True = remove Docker images after execution" ] }, { @@ -134,7 +105,7 @@ "id": "63f616a3", "metadata": {}, "source": [ - "# IERS Download (optionnal)" + "# IERS Download" ] }, { @@ -145,36 +116,38 @@ "outputs": [], "source": [ "# === DOWNLOAD IERS ===\n", - "\n", "import os\n", "import re\n", "import requests\n", "from datetime import datetime\n", "\n", - "DATA_DIR = os.path.join(WORKDIR, \"DATA\")\n", + "if len(PATH_IERS) != 0: #Safety in case the PATH_IERS isn't empty\n", + " print(\"Stopping because PATH_IERS is not empty\")\n", + " sys.exit(0)\n", "\n", - "if not os.path.exists(DATA_DIR):\n", - " raise RuntimeError(f\"DATA directory not found: {DATA_DIR}\")\n", + "if not os.path.exists(WORKDIR):\n", + " raise RuntimeError(f\"WORKDIR directory not found: {WORKDIR}\")\n", "\n", - "print(\"Bulletin output directory:\", DATA_DIR)\n", + "print(\"Bulletin output directory:\", WORKDIR)\n", "\n", + "PATH_IERS = WORKDIR # then our file will be there\n", "# =====================================================================\n", - "# 0. REMOVE EXISTING IERS BULLETINS\n", + "# REMOVE EXISTING IERS BULLETINS\n", "# =====================================================================\n", "\n", - "for f in os.listdir(DATA_DIR):\n", + "for f in os.listdir(WORKDIR):\n", " if f.startswith(\"bulletina-\") and f.endswith(\".txt\"):\n", - " os.remove(os.path.join(DATA_DIR, f))\n", + " os.remove(os.path.join(WORKDIR, f))\n", " print(\"Removed old bulletin A:\", f)\n", "\n", " if f.startswith(\"bulletinb-\") and f.endswith(\".txt\"):\n", - " os.remove(os.path.join(DATA_DIR, f))\n", + " os.remove(os.path.join(WORKDIR, f))\n", " print(\"Removed old bulletin B:\", f)\n", "\n", "print(\"Cleanup of old bulletins complete.\\n\")\n", "\n", "# =====================================================================\n", - "# 2. EXTRACT PRODUCT DATE (FROM DATASTRIP)\n", + "# EXTRACT PRODUCT DATE (FROM DATASTRIP)\n", "# =====================================================================\n", "\n", "datastrip_dir = os.path.join(PATH_L1B_DATA, \"DATASTRIP\")\n", @@ -244,7 +217,7 @@ "\n", " if response.status_code == 200:\n", " print(\"Bulletin found:\", index_str)\n", - " dest_file = os.path.join(DATA_DIR, f\"bulletina-{roman_year}-{index_str}.txt\")\n", + " dest_file = os.path.join(WORKDIR, f\"bulletina-{roman_year}-{index_str}.txt\")\n", " found = True\n", " break\n", "\n", @@ -288,10 +261,8 @@ "\n", "print(\"UserConf directory:\", USERCONF_DIR)\n", "\n", - "GEOID_DIR = os.path.join(WORKDIR, \"DATA\", \"GEOID\")\n", - "os.makedirs(GEOID_DIR, exist_ok=True)\n", "\n", - "print(\"Geoid directory:\", GEOID_DIR)\n", + "print(\"Geoid directory:\", PATH_GEOID)\n", "\n", "# =====================================================\n", "# 1. Extract mission from DATASTRIP\n", @@ -327,47 +298,29 @@ "# 3. Geoid management\n", "# =====================================================\n", "\n", - "# Notebook location (NOT relative to CWD)\n", - "notebook_dir = os.getcwd()\n", - "\n", - "# Relative path to DEM_GEOID from notebook\n", - "internal_geoid_dir = os.path.abspath(os.path.join(\n", - " notebook_dir,\n", - " \"..\", \"..\", \"src\", \"test\", \"resources\", \"DEM_GEOID\"\n", - "))\n", - "\n", - "# If WORKDIR/DATA/GEOID is empty -> copy internal files\n", - "if len(os.listdir(GEOID_DIR)) == 0:\n", - " print(\"GEOID directory is empty -> copying default geoid files...\")\n", - " for f in os.listdir(internal_geoid_dir):\n", - " src = os.path.join(internal_geoid_dir, f)\n", - " dst = os.path.join(GEOID_DIR, f)\n", - " shutil.copy(src, dst)\n", - "\n", - "# Detect .gtx inside GEOID_DIR\n", - "geoid_files = [f for f in os.listdir(GEOID_DIR) if f.lower().endswith(\".gtx\")]\n", + "# Detect .gtx inside PATH_GEOID\n", + "geoid_files = [f for f in os.listdir(PATH_GEOID) if f.lower().endswith(\".gtx\")]\n", "if len(geoid_files) == 0:\n", - " raise RuntimeError(\"No .gtx geoid file found in WORKDIR/DATA/GEOID.\")\n", + " raise RuntimeError(\"No .gtx geoid file found in PATH_GEOID.\")\n", "\n", - "docker_geoid = f\"/workspace/DATA/GEOID/{geoid_files[0]}\"\n", + "docker_geoid = f\"/data/GEOID/{geoid_files[0]}\"\n", "\n", "# =====================================================\n", "# 4. Locate IERS bulletin on host\n", "# =====================================================\n", "\n", - "DATA_DIR = os.path.join(WORKDIR, \"DATA\")\n", "iers_host = None\n", "\n", - "for f in os.listdir(DATA_DIR):\n", + "for f in os.listdir(PATH_IERS):\n", " if f.startswith(\"bulletin\"):\n", - " iers_host = os.path.join(DATA_DIR, f)\n", + " iers_host = os.path.join(PATH_IERS, f)\n", " break\n", "\n", "if iers_host is None:\n", - " raise RuntimeError(\"IERS bulletin not found inside WORKDIR/DATA directory.\")\n", - "\n", - "docker_iers = \"/workspace/DATA/\" + os.path.basename(iers_host)\n", + " raise RuntimeError(\"IERS bulletin not found inside PATH_IERS directory.\")\n", "\n", + "docker_iers = f\"/data/IERS/{os.path.basename(iers_host)}\"\n", + "print(docker_iers)\n", "# =====================================================\n", "# 5. Build config dictionary\n", "# =====================================================\n", @@ -527,6 +480,8 @@ "import os\n", "import subprocess\n", "\n", + "# Notebook location (NOT relative to CWD)\n", + "notebook_dir = os.getcwd()\n", "\n", "dockerfile_dir = os.path.abspath(os.path.join(\n", " notebook_dir,\n", @@ -562,7 +517,9 @@ " \"-v\", f\"{PATH_L1B_DATA}:/data/L1B\",\n", " \"-v\", f\"{PATH_DEM}:/data/DEM\",\n", " \"-v\", f\"{PATH_GIPP}:/data/GIPP\",\n", - " \"-v\", f\"{WORKDIR}:/workspace\", \n", + " \"-v\", f\"{PATH_GEOID}:/data/GEOID\",\n", + " \"-v\", f\"{PATH_IERS}:/data/IERS\",\n", + " \"-v\", f\"{WORKDIR}:/workspace\",\n", " \"sen2vm\",\n", " \"-c\", config_inside,\n", " \"-p\", params_inside\n", @@ -576,11 +533,13 @@ "# =====================================================\n", "# 3. REMOVE DOCKER IMAGE\n", "# =====================================================\n", - "\"\"\"\n", - "print(\"Removing Docker image 'sen2vm'...\")\n", - "subprocess.run([\"docker\", \"rmi\", \"-f\", \"sen2vm\"], check=True)\n", - "print(\"Docker image removed.\\n\")\n", - "\"\"\"\n" + "if REMOVE_DOCKER_IMAGE:\n", + " print(\"Removing Docker image 'sen2vm'...\")\n", + " subprocess.run([\"docker\", \"rmi\", \"-f\", \"sen2vm\"], check=True)\n", + " print(\"Docker images removed.\\n\")\n", + "else:\n", + " print(\"Docker images kept.\\n\")\n", + "\n" ] }, { @@ -609,8 +568,8 @@ "# =====================================================\n", "assert os.path.exists(PATH_L1B_DATA), \"L1B path missing\"\n", "\n", - "os.makedirs(INVERSE_OUTPUT_FOLDER, exist_ok=True)\n", - "assert os.path.exists(INVERSE_OUTPUT_FOLDER), \"Output folder missing\"\n", + "os.makedirs(OUTPUT_FOLDER, exist_ok=True)\n", + "assert os.path.exists(OUTPUT_FOLDER), \"Output folder missing\"\n", "\n", "# =====================================================\n", "# Locate product name\n", @@ -693,10 +652,10 @@ "# =====================================================\n", "# Output directories\n", "# =====================================================\n", - "os.makedirs(os.path.join(INVERSE_OUTPUT_FOLDER, \"GDAL_OUTPUT_ORTHO\"), exist_ok=True)\n", - "os.makedirs(os.path.join(INVERSE_OUTPUT_FOLDER, \"GDAL_OUTPUT_MOSAIC\"), exist_ok=True)\n", + "os.makedirs(os.path.join(OUTPUT_FOLDER, \"GDAL_OUTPUT_ORTHO\"), exist_ok=True)\n", + "os.makedirs(os.path.join(OUTPUT_FOLDER, \"GDAL_OUTPUT_MOSAIC\"), exist_ok=True)\n", "\n", - "print(\"Output folder:\", INVERSE_OUTPUT_FOLDER)\n", + "print(\"Output folder:\", OUTPUT_FOLDER)\n", "\n", "# =====================================================\n", "# Build GDAL docker\n", @@ -861,7 +820,7 @@ " \"--rm\",\n", " \"-v\", f\"{PATH_L1B_DATA}:/data/L1B\",\n", " \"-v\", f\"{WORKDIR}:/workspace\",\n", - " \"-v\", f\"{INVERSE_OUTPUT_FOLDER}:/output\",\n", + " \"-v\", f\"{OUTPUT_FOLDER}:/output\",\n", " \"gdal-latest\",\n", " \"/workspace/src/gdal_ortho.sh\"\n", "]\n", @@ -870,23 +829,18 @@ "subprocess.run(cmd_run, check=True)\n", "print(\"\\nGDAL ortho + mosaic complete.\\n\")\n", "\n", - "\"\"\"\n", + "\n", "# =====================================================\n", "# Cleanup docker image\n", "# =====================================================\n", - "print(\"Removing gdal-latest image...\\n\")\n", - "subprocess.run([\"docker\", \"rmi\", \"-f\", \"gdal-latest\"], check=True)\n", - "print(\"GDAL image removed.\\n\")\n", - "\"\"\"" + "\n", + "if REMOVE_DOCKER_IMAGE:\n", + " print(\"Removing gdal-latest image...\\n\")\n", + " subprocess.run([\"docker\", \"rmi\", \"-f\", \"gdal-latest\"], check=True)\n", + " print(\"GDAL image removed.\\n\")\n", + "else:\n", + " print(\"Docker images kept (faster next run, ~4 GB disk usage).\\n\")" ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "257b2ddd-8b9d-4c21-9233-0223c2a8b7d0", - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { From d80c536520582e1f83795b232bcedddf5de82165 Mon Sep 17 00:00:00 2001 From: paul-csgroup0411 Date: Fri, 22 May 2026 14:39:50 +0200 Subject: [PATCH 3/3] add of import sys in the IERS FILE donwload --- sen2vm-notebook/src/notebook.ipynb | 1 + 1 file changed, 1 insertion(+) diff --git a/sen2vm-notebook/src/notebook.ipynb b/sen2vm-notebook/src/notebook.ipynb index 4ef98bce..40b50637 100644 --- a/sen2vm-notebook/src/notebook.ipynb +++ b/sen2vm-notebook/src/notebook.ipynb @@ -118,6 +118,7 @@ "# === DOWNLOAD IERS ===\n", "import os\n", "import re\n", + "import sys\n", "import requests\n", "from datetime import datetime\n", "\n",