Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -25,21 +25,21 @@ repos:

# Python file formatting
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.15.4
rev: v0.15.9
hooks:
- id: ruff
args: [--fix, --exit-non-zero-on-fix]
- id: ruff-format

# Snakemake file formatting
- repo: https://github.com/snakemake/snakefmt
rev: v0.11.4
rev: v1.0.0
hooks:
- id: snakefmt

# Spelling
- repo: https://github.com/codespell-project/codespell
rev: v2.4.1
rev: v2.4.2
hooks:
- id: codespell
files: .*\.(py|smk|md)$|^Snakefile$
Expand Down
12 changes: 6 additions & 6 deletions tests/integration/Snakefile
Original file line number Diff line number Diff line change
Expand Up @@ -7,17 +7,17 @@ configfile: workflow.source_path("./test_config.yaml")
# `config`: specifies the module configuration.
# `pathvars:` helps you re-wire where the module places files.
module module_geo_boundaries:
snakefile:
"../../workflow/Snakefile"
config:
config["module_geo_boundaries"]
pathvars:
# Redirect specific module results (outputs)
shapes="results/outputs/shapes.parquet",
# Redirect module intermediate files
logs="resources/geo_boundaries/logs",
resources="resources/geo_boundaries/resources",
results="resources/geo_boundaries/results",
snakefile:
"../../workflow/Snakefile"
config:
config["module_geo_boundaries"]


# Add a prefix to all module rules, to avoid naming conflicts.
Expand All @@ -26,8 +26,8 @@ use rule * from module_geo_boundaries as geo_boundaries_module_*

# Request something from the module
rule all:
message:
"A generic test case for this module."
default_target: True
input:
"results/outputs/shapes.parquet",
message:
"A generic test case for this module."
4 changes: 2 additions & 2 deletions workflow/Snakefile
Original file line number Diff line number Diff line change
Expand Up @@ -35,14 +35,14 @@ workflow.source_path("scripts/_schemas.py")


rule all:
message:
"ERROR: Invalid `rule all:` call"
default_target: True
output:
"INVALID",
log:
stderr="<logs>/all.stderr",
conda:
"envs/shell.yaml"
message:
"ERROR: Invalid `rule all:` call"
shell:
'echo "This workflow must be called as a snakemake module." > {log.stderr}'
42 changes: 21 additions & 21 deletions workflow/rules/automatic.smk
Original file line number Diff line number Diff line change
Expand Up @@ -5,23 +5,21 @@ Small transformations might be performed to make the data easier to work with.


rule download_country_overture:
message:
"Download '{wildcards.country}_{wildcards.subtype}' dataset from Overture Maps."
params:
version=config["overture_release"],
output:
path="<resources>/automatic/countries/overture_{country}_{subtype}.parquet",
log:
"<logs>/{country}/download_country_overture_{subtype}.log",
conda:
"../envs/shape.yaml"
params:
version=config["overture_release"],
message:
"Download '{wildcards.country}_{wildcards.subtype}' dataset from Overture Maps."
script:
"../scripts/download_country_overture.py"


rule download_country_gadm:
message:
"Download '{wildcards.country}_{wildcards.subtype}' dataset from GADM."
output:
path=temp(
"<resources>/automatic/countries/raw_gadm_{country}_{subtype}.parquet"
Expand All @@ -30,16 +28,13 @@ rule download_country_gadm:
"<logs>/{country}/download_country_gadm_{subtype}.log",
conda:
"../envs/shape.yaml"
message:
"Download '{wildcards.country}_{wildcards.subtype}' dataset from GADM."
script:
"../scripts/download_country_gadm.py"


rule standardise_country_gadm:
message:
"Standardise '{wildcards.country}_{wildcards.subtype}' GADM dataset."
params:
country_id=lambda wc: str(wc.country),
subtype=lambda wc: str(wc.subtype),
input:
raw="<resources>/automatic/countries/raw_gadm_{country}_{subtype}.parquet",
output:
Expand All @@ -48,30 +43,31 @@ rule standardise_country_gadm:
"<logs>/{country}/standardise_country_gadm_{subtype}.log",
conda:
"../envs/shape.yaml"
params:
country_id=lambda wc: str(wc.country),
subtype=lambda wc: str(wc.subtype),
message:
"Standardise '{wildcards.country}_{wildcards.subtype}' GADM dataset."
script:
"../scripts/standardise_country_gadm.py"


rule download_nuts:
message:
"Download '{wildcards.resolution}_{wildcards.year}_{wildcards.level}' from NUTS."
params:
epsg=internal["nuts"]["epsg"],
output:
path="<resources>/automatic/nuts/nuts_{resolution}_{year}_{level}.parquet",
log:
"<logs>/download_nuts_{resolution}_{year}_{level}.log",
conda:
"../envs/shape.yaml"
params:
epsg=internal["nuts"]["epsg"],
message:
"Download '{wildcards.resolution}_{wildcards.year}_{wildcards.level}' from NUTS."
script:
"../scripts/download_nuts.py"


rule standardise_country_nuts:
message:
"Standardise '{wildcards.country}_{wildcards.subtype}' NUTS dataset."
params:
year=lambda wc: config["countries"][wc.country]["year"],
input:
raw=lambda wc: f"<resources>/automatic/nuts/nuts_{config["countries"][wc.country]["resolution"]}_{config["countries"][wc.country]["year"]}_{wc.subtype}.parquet",
output:
Expand All @@ -80,19 +76,23 @@ rule standardise_country_nuts:
"<logs>/{country}/standardise_country_nuts_{subtype}.log",
conda:
"../envs/shape.yaml"
params:
year=lambda wc: config["countries"][wc.country]["year"],
message:
"Standardise '{wildcards.country}_{wildcards.subtype}' NUTS dataset."
script:
"../scripts/standardise_country_nuts.py"


rule download_marine_eez_area:
message:
"Download and standardise '{wildcards.country}' EEZ dataset."
output:
path="<resources>/automatic/eez/{country}.parquet",
plot="<resources>/automatic/eez/{country}.png",
log:
"<logs>/{country}/download_marine_eez_area.log",
conda:
"../envs/shape.yaml"
message:
"Download and standardise '{wildcards.country}' EEZ dataset."
script:
"../scripts/download_marine_eez_area.py"
8 changes: 4 additions & 4 deletions workflow/rules/build.smk
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,6 @@


rule build_combined_area:
message:
"Combine land and marine polygons."
params:
crs=config["crs"],
input:
countries=[
f"<resources>/automatic/countries/{data['source']}_{country}_{data['subtype']}.parquet"
Expand All @@ -26,5 +22,9 @@ rule build_combined_area:
"<logs>/build_combined_area.log",
conda:
"../envs/shape.yaml"
params:
crs=config["crs"],
message:
"Combine land and marine polygons."
script:
"../scripts/build_combined_area.py"
Loading