ci: Switch test data to rclone, tune Polars threading, and clean up coverage filtering

.github/workflows/main.yml

@@ -14,6 +14,6 @@ jobs:
       - uses: actions/checkout@v4
       - uses: prefix-dev/setup-pixi@v0.8.1
         with:
-          pixi-version: v0.37.0
+          pixi-version: v0.70.2
           cache: true
       - run: pixi run test

pixi.toml

@@ -1,4 +1,4 @@
-[project]
+[workspace]
 authors = ["Mitchell Robert Vollger <mrvollger@gmail.com>"]
 channels = ["conda-forge", "bioconda"]
 description = "Add a short description here"
@@ -9,52 +9,48 @@ version = "0.1.2"
 [tasks]
 fmt = "ruff format . && taplo format pixi.toml && snakefmt workflow/"
 test-data = { cmd = [
-    "cd",
-    "$INIT_CWD",
-    "&&",
-    "mkdir",
-    "-p",
-    "fire-test-data",
-    "&&",
-    "aws",
-    "s3",
-    "--no-sign-request",
-    "sync",
-    "--endpoint-url",
-    "https://s3.kopah.orci.washington.edu",
-    "s3://stergachis/public/FIRE/test-data",
-    "fire-test-data/",
+  "cd",
+  "$INIT_CWD",
+  "&&",
+  "mkdir",
+  "-p",
+  "fire-test-data",
+  "&&",
+  "rclone",
+  "sync",
+  ":s3,env_auth=false,provider=Other,endpoint=s3.kopah.orci.washington.edu:stergachis/public/FIRE/test-data",
+  "fire-test-data/",
 ] }
 test = { cmd = [
-    "cd",
-    "$INIT_CWD/fire-test-data",
-    "&&",
-    "snakemake",
-    "-s",
-    "$PIXI_PROJECT_ROOT/workflow/Snakefile",
-    "--configfile",
-    "test.yaml",
-    "-k",
+  "cd",
+  "$INIT_CWD/fire-test-data",
+  "&&",
+  "snakemake",
+  "-s",
+  "$PIXI_PROJECT_ROOT/workflow/Snakefile",
+  "--configfile",
+  "test.yaml",
+  "-k",
 ], depends-on = [
-    "test-data",
+  "test-data",
 ], clean-env = true }
 fire = { cmd = [
-    "cd",
-    "$INIT_CWD",
-    "&&",
-    "snakemake",
-    "-s",
-    "$PIXI_PROJECT_ROOT/workflow/Snakefile",
+  "cd",
+  "$INIT_CWD",
+  "&&",
+  "snakemake",
+  "-s",
+  "$PIXI_PROJECT_ROOT/workflow/Snakefile",
 ] }
 slurm = { cmd = [
-    "cd",
-    "$INIT_CWD",
-    "&&",
-    "snakemake",
-    "-s",
-    "$PIXI_PROJECT_ROOT/workflow/Snakefile",
-    "--profile",
-    "$PIXI_PROJECT_ROOT/profiles/slurm-executor",
+  "cd",
+  "$INIT_CWD",
+  "&&",
+  "snakemake",
+  "-s",
+  "$PIXI_PROJECT_ROOT/workflow/Snakefile",
+  "--profile",
+  "$PIXI_PROJECT_ROOT/profiles/slurm-executor",
 ] }
 
 [dependencies]
@@ -63,7 +59,7 @@ snakemake = "==8.21"
 snakemake-executor-plugin-slurm = ">=0.11.2"
 snakefmt = "*"
 ruff = "*"
-awscli = "2.22.*"
+rclone = "*"
 taplo = "*"
 
 [pypi-dependencies]

workflow/Snakefile

@@ -23,7 +23,7 @@ FULL_VERSION = f"v{os.environ.get("PIXI_PROJECT_VERSION", "UNK")}"
 # trim the FULL_VERSION to just the just the major and minor version
 VERSION = ".".join(FULL_VERSION.split(".")[:2])
 
-# use the full version if requested 
+# use the full version if requested
 if config.get("full-version", False):
     VERSION = FULL_VERSION
 
@@ -130,11 +130,8 @@ rule all:
         # coverage information
         expand(rules.coverage.output, sm=MANIFEST.index, v=VERSION),
         expand(rules.exclude_from_shuffle.output, sm=MANIFEST.index, v=VERSION),
-        expand(rules.unreliable_coverage_regions.output, sm=MANIFEST.index, v=VERSION),
         expand(
-            rules.pileup.output.bed,
-            sm=MANIFEST.index,
-            v=VERSION,
+            rules.unreliable_coverage_regions.output.bed, sm=MANIFEST.index, v=VERSION
         ),
         # model results
         expand(rules.fire.output.cram, sm=MANIFEST.index, v=VERSION),

workflow/rules/coverages.smk

@@ -17,7 +17,7 @@ rule genome_bedgraph:
         DEFAULT_ENV
     shell:
         """ 
-        mosdepth -f {input.ref} -t {threads} tmp {input.cram}
+        mosdepth -F 4 -f {input.ref} -t {threads} tmp {input.cram}
         bgzip -cd tmp.per-base.bed.gz \
             | LC_ALL=C sort --parallel={threads} -k1,1 -k2,2n -k3,3n -k4,4  \
             | bgzip -@ {threads} \

workflow/rules/fire-peaks.smk

@@ -76,10 +76,12 @@ rule fdr_table:
         "../envs/python.yaml"
     params:
         script=workflow.source_path("../scripts/fdr-table.py"),
+    threads: 16
     resources:
         mem_mb=get_mem_mb_xl,
     shell:
         """
+        export POLARS_MAX_THREADS={threads}
         MIN=$(cat {input.minimum})
         MAX=$(cat {input.maximum})
         python {params.script} -v 1 {input.shuffled} {output.tbl} --max-cov $MAX --min-cov $MIN
@@ -124,6 +126,7 @@ rule fdr_track_chromosome:
         mem_mb=get_mem_mb_xl,
     shell:
         """
+        export POLARS_MAX_THREADS={threads}
         python {params.script} -v 1 \
             --fdr-table {input.fdr_tbl} \
             {input.pileup} {output.bed}
@@ -141,7 +144,7 @@ rule pileup:
         fofn=temp("temp/{sm}/fire/fire-{v}-pileup.fofn"),
         bed="results/{sm}/{sm}-fire-{v}-pileup.bed.gz",
         tbi="results/{sm}/{sm}-fire-{v}-pileup.bed.gz.tbi",
-    threads: 4
+    threads: 8
     conda:
         DEFAULT_ENV
     shell:
@@ -231,6 +234,7 @@ rule fdr_peaks_by_fire_elements_chromosome:
         min_frac_accessible=MIN_FRAC_ACCESSIBLE,
     shell:
         """
+        export POLARS_MAX_THREADS={threads}
         bgzip -cd {input.bed} \
             | python {params.script} -v 1 \
                 --max-cov $(cat {input.maximum}) \

workflow/scripts/cov.py

@@ -1,3 +1,7 @@
+import os
+
+os.environ["POLARS_MAX_THREADS"] = str(snakemake.threads)
+
 import pandas as pd
 import sys
 import math