scripts/deploy_all_pipelines.sh

@@ -92,12 +92,10 @@ define_paths() {
   ## resources
   config_file=$repo_path/static/paths.cfg
   pipeline_table=$repo_path/static/pipelines.tsv
-  cephfs_path_table=$repo_path/static/ceph_db_symlinks.tsv
 
   ## checks
   [[ ! -f $config_file ]] && die "File not found: $config_file"
   [[ ! -f $pipeline_table ]] && die "File not found: $pipeline_table"
-  [[ ! -f $cephfs_path_table ]] && die "File not found: $cephfs_path_table"
 
   ## source a config file
   source "$config_file"
@@ -143,9 +141,10 @@ pipeline_table_column2array() {
 deploy_single_pipeline() {
   ## [Rationale] deploy_single_pipeline() sequential switch logic: skip is by default on true and when specific condition is met, the switch is set to the value of dryrun, hence dryrun overrides
 
-  local pipeline branch pipeline_dir skip_git_pull skip_mamba_install symlink_target symlink_path
+  local pipeline branch pipeline_dir skip_git_pull skip_setup_env skip_setup_data
   skip_git_pull=true
-  skip_mamba_install=true
+  skip_setup_env=true
+  skip_setup_data=true
 
   pipeline=$1
   branch=${2:-main}
@@ -169,18 +168,14 @@ branch:|$branch
   )
 
   ## install mamba environment
-  [[ ! "${conda_envs[*]}" =~ ${pipeline,,} && -f "$pipeline_dir/${pipeline_setup_sh[$pipeline]}" ]] && skip_mamba_install=$dryrun || logwarn "Conda environment already exists or setup script not found"
-  [[ $skip_mamba_install == false ]] && bash "$pipeline_dir"/"${pipeline_setup_sh[$pipeline]}" --auto --mamba
+  [[ ! "${conda_envs[*]}" =~ ${pipeline,,} && -f "$pipeline_dir/${pipeline_setup_sh[$pipeline]}" ]] && skip_setup_env=$dryrun || logwarn "Conda environment already exists or setup script not found"
+  [[ $skip_setup_env == false ]] && bash "$pipeline_dir"/"${pipeline_setup_sh[$pipeline]}" --auto ${setup_args_env[$pipeline]}
 
-  ## symlink to cephfs
-  mapfile -t symlink_target < <(grep -P "^$pipeline" "$cephfs_path_table" | awk '{print $2}')
-  mapfile -t symlink_path < <(grep -P "^$pipeline" "$cephfs_path_table" | awk '{print $3}')
-
-  for i in "${!symlink_target[@]}"; do
-    stat "$pipeline_dir/${symlink_path[$i]}" &> /dev/null && logwarn "Symlink already exists: $pipeline_dir/${symlink_path[$i]}" && continue
-    logexec "ln -s ${symlink_target[$i]} $pipeline_dir/${symlink_path[$i]}"
-  done
+  ## install mamba environment
+  [[ -f "$pipeline_dir/${pipeline_setup_sh[$pipeline]}" ]] && skip_setup_data=$dryrun || logwarn "Conda environment already exists or setup script not found"
+  [[ $skip_setup_data == false ]] && bash "$pipeline_dir"/"${pipeline_setup_sh[$pipeline]}" --auto ${setup_args_data[$pipeline]}
 
+  logsuccess "Pipeline $pipeline deployed successfully"
   return 0
 }
 
@@ -198,6 +193,14 @@ deploy_bfr_abc() {
   declare -g -A pipeline_setup_sh
   for key in "${!column_array[@]}"; do pipeline_setup_sh["$key"]="${column_array[$key]}"; done
 
+  pipeline_table_column2array "setup_args_env"
+  declare -g -A setup_args_env
+  for key in "${!column_array[@]}"; do setup_args_env["$key"]="${column_array[$key]}"; done
+
+  pipeline_table_column2array "setup_args_data"
+  declare -g -A setup_args_data
+  for key in "${!column_array[@]}"; do setup_args_data["$key"]="${column_array[$key]}"; done
+
   logheader "Git Remote Origin URL:"
   column -t -s "|" < <(for key in "${!repository_origin_urls[@]}"; do echo "$key|${repository_origin_urls[$key]}"; done)
 

scripts/symlinks_for_harmonization.sh

@@ -34,7 +34,7 @@ parse_args() {
   local args=("$@")
 
   ## default values of variables
-  repo_path=$(dirname $script_path)
+  repo_path=$(dirname "$script_path")
   mode="query"
 
   ## default values of switches
@@ -73,8 +73,8 @@ parse_args() {
 
 ## Source Helper Functions --------------------------------------------------
 script_real=$(realpath -P "${BASH_SOURCE[0]}")
-script_path=$(dirname $script_real)
-script_name=$(basename $script_real)
+script_path=$(dirname "$script_real")
+script_name=$(basename "$script_real")
 [[ ! -f "$script_path/helper_functions.sh" ]] && echo "[ERROR] File not found: $script_path/helper_functions.sh" && kill -s TERM $$
 # shellcheck source=./helper_functions.sh
 source "$script_path/helper_functions.sh"
@@ -156,7 +156,7 @@ create_relative_symlink() {
   [[ ! -f $target ]] && logerror "The target directory does not exist: $target_dir" && return 0
 
   # create link
-  logexec "cd \"$symlink_dir\"; ln -sf \"$target\" \"$(basename $symlink)\""
+  logexec "cd \"$symlink_dir\"; ln -sf \"$target\" \"$(basename "$symlink")\""
 }
 
 create_absolute_symlink() {
@@ -175,7 +175,7 @@ create_absolute_symlink() {
   [[ ! -f $target ]] && logerror "The target directory does not exist: $target_dir" && return 0
 
   # create link
-  logexec "cd \"$symlink_dir\"; ln -sf \"$target\" \"$(basename $symlink)\""
+  logexec "cd \"$symlink_dir\"; ln -sf \"$target\" \"$(basename "$symlink")\""
 }
 
 create_absolute_symlinks_from_array() {

static/ceph_db_symlinks.tsv

-pipeline	target	symlink
-aquamis	/cephfs/abteilung4/NGS/Pipelines/aquamis_akkreditierung/reference_db/confindr	reference_db/confindr
-aquamis	/cephfs/abteilung4/NGS/ReferenceDB/kraken2/minikraken2_v1_8GB	reference_db/kraken
-aquamis	/cephfs/abteilung4/NGS/Pipelines/aquamis_akkreditierung/reference_db/mash/mashDB.msh	reference_db/mash/mashDB.msh
-aquamis	/cephfs/abteilung4/NGS/Pipelines/aquamis/reference_db/mash/genomes	reference_db/mash/genomes
-aquamis	/cephfs/abteilung4/NGS/Pipelines/aquamis_akkreditierung/reference_db/taxonkit	reference_db/taxonkit
-bakcharak	/cephfs/abteilung4/NGS/ReferenceDB/bakcharak/bakta	databases/bakta
-bakcharak	/cephfs/abteilung4/NGS/ReferenceDB/bakcharak/mash/genomes	databases/mash/genomes
-bakcharak	/cephfs/abteilung4/NGS/ReferenceDB/bakcharak/mash/plasmids	databases/mash/plasmids
-bakcharak	/cephfs/abteilung4/NGS/ReferenceDB/bakcharak/platon	databases/platon
-bakcharak	/cephfs/abteilung4/NGS/ReferenceDB/bakcharak/plasmidblast	databases/plasmidblast
-chewiesnake	/home/NGSAdmin/HPC_referencedb/cgmlst/active_schemes
-milonga	/cephfs/abteilung4/NGS/ReferenceDB/kraken2/minikraken2_v1_8GB	databases/kraken2
-milonga	/cephfs/abteilung4/NGS/Pipelines/milonga/databases/platon	databases/kraken2

static/pipelines.tsv

-pipeline	official_name	repository_name	repository_origin_url	main_snakefile	pipeline_py	pipeline_setup_sh	pipeline_wrapper_sh	install_args_full
-aquamis	AQUAMIS	AQUAMIS	https://gitlab.com/bfr_bioinformatics/AQUAMIS.git	Snakefile	aquamis.py	scripts/aquamis_setup.sh	scripts/aquamis_wrapper.sh	--mamba --busco --databases --test_data
-bakcharak	BakCharak	bakcharak	https://gitlab.com/bfr_bioinformatics/bakcharak.git	Snakefile	bakcharak.py	scripts/bakcharak_setup.sh	scripts/bakcharak_wrapper.sh	--mamba --amrfinder_update --databases --bakta light --test_data
-chewiesnake	ChewieSnake	chewieSnake	https://gitlab.com/bfr_bioinformatics/chewieSnake.git	chewieSnake_fromcontigs.smk	chewieSnake.py	scripts/chewieSnake_setup.sh	scripts/chewieSnake_wrapper.sh	--mamba --databases
-snippysnake	snippySnake	snippySnake	https://gitlab.com/bfr_bioinformatics/snippySnake.git	snippySnake.smk	snippySnake.py	scripts/snippySnake_setup.sh	scripts/snippySnake_wrapper.sh	--mamba --databases --test_data
-milonga	MiLongA	milonga	https://gitlab.com/bfr_bioinformatics/milonga.git	pipeline.smk	milonga.py	scripts/milonga_setup.sh	scripts/milonga_wrapper.sh	--mamba --test_data --sra_toolkit
+pipeline	official_name	repository_name	repository_origin_url	main_snakefile	pipeline_py	pipeline_setup_sh	pipeline_wrapper_sh	setup_args_env	setup_args_data
+aquamis	AQUAMIS	AQUAMIS	https://gitlab.com/bfr_bioinformatics/AQUAMIS.git	Snakefile	aquamis.py	scripts/aquamis_setup.sh	scripts/aquamis_wrapper.sh	--mamba --busco	--symlinks_to_ceph --test_data
+bakcharak	BakCharak	bakcharak	https://gitlab.com/bfr_bioinformatics/bakcharak.git	Snakefile	bakcharak.py	scripts/bakcharak_setup.sh	scripts/bakcharak_wrapper.sh	--mamba --amrfinder_update	--symlinks_to_ceph --test_data
+chewiesnake	ChewieSnake	chewieSnake	https://gitlab.com/bfr_bioinformatics/chewieSnake.git	chewieSnake_fromcontigs.smk	chewieSnake.py	scripts/chewieSnake_setup.sh	scripts/chewieSnake_wrapper.sh	--mamba	--dryrun
+snippysnake	snippySnake	snippySnake	https://gitlab.com/bfr_bioinformatics/snippySnake.git	snippySnake.smk	snippySnake.py	scripts/snippySnake_setup.sh	scripts/snippySnake_wrapper.sh	--mamba	--test_data --sra_toolkit
+milonga	MiLongA	milonga	https://gitlab.com/bfr_bioinformatics/milonga.git	pipeline.smk	milonga.py	scripts/milonga_setup.sh	scripts/milonga_wrapper.sh	--mamba	--symlinks_to_ceph --test_data