// Define the scratch directory, which will be used for storing the nextflow
// work directory and for caching apptainer/singularity files.
// Default to /tmp directory if $VSC_SCRATCH scratch env is not available,
// see: https://github.com/nf-core/configs?tab=readme-ov-file#adding-a-new-config
def scratch_dir = System.getenv("VSC_SCRATCH") ?: "/tmp"
 
// Specify the work directory.
workDir = "$scratch_dir/work"
 
// Perform work directory cleanup when the run has succesfully completed.
cleanup = true
 
def host = System.getenv("VSC_INSTITUTE")
 
// Check if APPTAINER_TMPDIR/SINGULARITY_TMPDIR environment variables are set.
// If they are available, try to create the tmp directory at the specified location.
// Skip if host is not CalcUA to avoid hindering github actions.
if ( host == "antwerpen" ) {
    def apptainer_tmpdir = System.getenv("APPTAINER_TMPDIR") ?: System.getenv("SINGULARITY_TMPDIR") ?: null
    if (! apptainer_tmpdir ) {
        def tmp_dir = System.getenv("TMPDIR") ?: "/tmp"
        System.err.println("\nWARNING: APPTAINER_TMPDIR/SINGULARITY_TMPDIR environment variable was not found.\nPlease add the line 'export APPTAINER_TMPDIR=\"\${VSC_SCRATCH}/apptainer/tmp\"' to your ~/.bashrc file (or set it with sbatch or in your job script).\nDefaulting to local $tmp_dir on the execution node of the Nextflow head process.\n")
        // TODO: check if images stored there can be accessed by slurm jobs on other nodes
    } else {
        apptainer_tmpdir = new File(apptainer_tmpdir)
        if (! apptainer_tmpdir.exists() ) {
            try {
                dir_created = apptainer_tmpdir.mkdirs()
            } catch (java.io.IOException e) {
                System.err.println("\nWARNING: Could not create directory at the location specified by APPTAINER_TMPDIR/SINGULARITY_TMPDIR: $apptainer_tmpdir\nPlease check if this is a valid path to which you have write permission. Exiting...\n")
            }
        }
    }
}
 
// Function to check if the selected partition profile matches the partition on which the master
// nextflow job was launched (either implicitly or via `sbatch --partition=<partition-name>`).
// If the profile type is `*_local` and the partitions do not match, stop the execution and
// warn the user.
def partition_checker(String profile) {
    // Skip check if host machine is not CalcUA, in order to not hinder github actions.
    if ( host != "antwerpen" ) {
        // System.err.println("\nWARNING: Skipping comparison of current partition and requested profile because the current machine is not VSC CalcUA.")
        return
    }
 
    def current_partition = System.getenv("SLURM_JOB_PARTITION")
 
    try {
        current_partition
    } catch (java.io.IOException e) {
        System.err.println("\nWARNING: Current partition could not be found in the expected \$SLURM_JOB_PARTITION environment variable. Please make sure that you submit your pipeline via a Slurm job instead of running the nextflow command directly on a login node.\nExiting...\n")
    }
 
    try {
        current_partition = profile
    } catch (java.io.IOException e) {
        System.err.println("\nWARNING: Slurm job was launched on the \'$current_partition\' partition, but the selected nextflow profile points to the $profile partition instead ('${profile}_local'). When using one of the local node execution profiles, please launch the job on the corresponding partition in Slurm.\nE.g., Slurm job submission command:\n    sbatch --account <project_account> --partition=broadwell script.slurm\nand job script containing a nextflow command with matching profile section:\n    nextflow run <pipeline> -profile vsc_calcua,broadwell_local\nExiting...\n")
    }
}
 
// Reduce the job submit rate to about 30 per minute, this way the server
// won't be bombarded with jobs.
// Limit queueSize to keep job rate under control and avoid timeouts.
// Set read timeout to the maximum wall time.
// See: https://www.nextflow.io/docs/latest/config.html#scope-executor
executor {
    submitRateLimit = '30/1min'
    queueSize = 10
    exitReadTimeout = 7.day
}
 
// Add backoff strategy to catch cluster timeouts and proper symlinks of files in scratch
// to the work directory.
// See: https://www.nextflow.io/docs/latest/config.html#scope-process
process {
    stageInMode = "symlink"
    stageOutMode = "rsync"
    errorStrategy = { sleep(Math.pow(2, task.attempt ?: 1) * 200 as long); return 'retry' }
    maxRetries = 3
}
 
// Specify that apptainer/singularity should be used and where the cache dir will be for the images.
// The singularity directive is used in favour of the apptainer one, because currently the apptainer
// variant will pull in (and convert) docker images, instead of using pre-built singularity ones.
// To use the pre-built singularity containers instead, the singularity options should be selected
// with apptainer installed on the system, which defines singularity as an alias (as is the case
// on CalcUA).
// See https://nf-co.re/docs/usage/installation#pipeline-software
// and https://nf-co.re/tools#how-the-singularity-image-downloads-work
// See https://www.nextflow.io/docs/latest/config.html#scope-singularity
singularity {
    enabled = true
    autoMounts = true
    // See https://www.nextflow.io/docs/latest/singularity.html#singularity-docker-hub
    cacheDir = "$scratch_dir/apptainer/nextflow_cache"  // Equivalent to setting NXF_APPTAINER_CACHEDIR/NXF_SINGULARITY_CACHEDIR environment variable
}
 
// Define profiles for the following partitions:
// - zen2, zen3, zen3_512 (Vaughan)
// - broadwell, broadwell_256 (Leibniz)
// - skylake (Breniac, formerly Hopper)
// For each partition, there is a "*_slurm" profile and a "*_local" profile.
// The former uses the slurm executor to submit each nextflow task as a separate job,
// whereas the latter runs all tasks on the individual node on which the nextflow
// master process was launched.
// See: https://www.nextflow.io/docs/latest/config.html#config-profiles
profiles {
    // Automatic slurm partition selection based on task requirements
    slurm {
        params {
            config_profile_description = 'Slurm profile with automatic partition selection for use on the CalcUA VSC HPC.'
            config_profile_contact = 'pmoris@itg.be (GitHub: @pmoris)'
            config_profile_url = 'https://docs.vscentrum.be/antwerp/tier2_hardware.html'
            max_memory = 496.GB // = max memory of high memory nodes
            max_cpus = 64   // = cpu count of largest nodes
            max_time = 7.day    // wall time of longest running nodes
        }
        process {
            executor = 'slurm'
            queue = {
                // long running
                if ( task.time > 3.day ) {
                    'skylake'
                // high memory
                } else if ( task.memory > 240.GB ) {
                    'zen3_512'
                // medium memory and high cpu
                } else if ( task.memory > 112.GB && task.cpus > 28 ) {
                    'zen2,zen3'
                // medium memory and lower cpu
                } else if ( task.memory > 112.GB && task.cpus < 28 ) {
                    'broadwell_256,zen2,zen3'
                // lower memory and high cpu
                } else if ( task.cpus > 28 ) {
                    'zen2,zen3'
                // lower memory and lower cpu
                } else {
                    'broadwell,skylake,zen2,zen3'
                }
            }
        }
    }
    // Vaughan partitions
    zen2_slurm {
        params {
            config_profile_description = 'Zen2 Slurm profile for use on the Vaughan cluster of the CalcUA VSC HPC.'
            config_profile_contact = 'pmoris@itg.be (GitHub: @pmoris)'
            config_profile_url = 'https://docs.vscentrum.be/antwerp/tier2_hardware/vaughan_hardware.html'
            max_memory = 240.GB // 256 GB (total) - 16 GB (buffer)
            max_cpus = 64
            max_time = 3.day
        }
        process {
            executor = 'slurm'
            queue = 'zen2'
        }
    }
    zen2_local {
        params {
            config_profile_description = 'Zen2 local profile for use on a single node of the Vaughan cluster of the CalcUA VSC HPC.'
            config_profile_contact = 'pmoris@itg.be (GitHub: @pmoris)'
            config_profile_url = 'https://docs.vscentrum.be/antwerp/tier2_hardware/vaughan_hardware.html'
            max_memory = get_allocated_mem(240) // 256 GB (total) - 16 GB (buffer)
            max_cpus = get_allocated_cpus(64)
            max_time = 3.day
        }
        process {
            executor = 'local'
        }
        partition_checker("zen2")
    }
    zen3_slurm {
        params {
            config_profile_description = 'Zen3 Slurm profile for use on the Vaughan cluster of the CalcUA VSC HPC.'
            config_profile_contact = 'pmoris@itg.be (GitHub: @pmoris)'
            config_profile_url = 'https://docs.vscentrum.be/antwerp/tier2_hardware/vaughan_hardware.html'
            max_memory = 240.GB // 256 GB (total) - 16 GB (buffer)
            max_cpus = 64
            max_time = 3.day
        }
        process {
            executor = 'slurm'
            queue = 'zen3'
        }
    }
    zen3_local {
        params {
            config_profile_description = 'Zen3 local profile for use on a single node of the Vaughan cluster of the CalcUA VSC HPC.'
            config_profile_contact = 'pmoris@itg.be (GitHub: @pmoris)'
            config_profile_url = 'https://docs.vscentrum.be/antwerp/tier2_hardware/vaughan_hardware.html'
            max_memory = get_allocated_mem(240) // 256 GB (total) - 16 GB (buffer)
            max_cpus = get_allocated_cpus(64)
            max_time = 3.day
        }
        process {
            executor = 'local'
        }
        partition_checker("zen3")
    }
    zen3_512_slurm {
        params {
            config_profile_description = 'Zen3_512 Slurm profile for use on the Vaughan cluster of the CalcUA VSC HPC.'
            config_profile_contact = 'pmoris@itg.be (GitHub: @pmoris)'
            config_profile_url = 'https://docs.vscentrum.be/antwerp/tier2_hardware/vaughan_hardware.html'
            max_memory = 496.GB // 512 GB (total) - 16 GB (buffer)
            max_cpus = 64
            max_time = 3.day
        }
        process {
            executor = 'slurm'
            queue = 'zen3_512'
        }
    }
    zen3_512_local {
        params {
            config_profile_description = 'Zen3_512 local profile for use on a single node of the Vaughan cluster of the CalcUA VSC HPC.'
            config_profile_contact = 'pmoris@itg.be (GitHub: @pmoris)'
            config_profile_url = 'https://docs.vscentrum.be/antwerp/tier2_hardware/vaughan_hardware.html'
            max_memory = get_allocated_mem(496) // 512 GB (total) - 16 GB (buffer)
            max_cpus = get_allocated_cpus(64)
            max_time = 3.day
        }
        process {
            executor = 'local'
        }
        partition_checker("zen3_512")
    }
    // Leibniz partitions
    broadwell_slurm {
        params {
            config_profile_description = 'Broadwell Slurm profile for use on the Leibniz cluster of the CalcUA VSC HPC.'
            config_profile_contact = 'pmoris@itg.be (GitHub: @pmoris)'
            config_profile_url = 'https://docs.vscentrum.be/antwerp/tier2_hardware/leibniz_hardware.html'
            max_memory = 112.GB // 128 GB (total) - 16 GB (buffer)
            max_cpus = 28
            max_time = 3.day
        }
        process {
            executor = 'slurm'
            queue = 'broadwell'
        }
    }
    broadwell_local {
        params {
            config_profile_description = 'Broadwell local profile for use on a single node of the Leibniz cluster of the CalcUA VSC HPC.'
            config_profile_contact = 'pmoris@itg.be (GitHub: @pmoris)'
            config_profile_url = 'https://docs.vscentrum.be/antwerp/tier2_hardware/leibniz_hardware.html'
            max_memory = get_allocated_mem(112) // 128 GB (total) - 16 GB (buffer)
            max_cpus = get_allocated_cpus(28)
            max_time = 3.day
        }
        process {
            executor = 'local'
        }
        partition_checker("broadwell")
    }
    broadwell_256_slurm {
        params {
            config_profile_description = 'Broadwell_256 Slurm profile for use on the Leibniz cluster of the CalcUA VSC HPC.'
            config_profile_contact = 'pmoris@itg.be (GitHub: @pmoris)'
            config_profile_url = 'https://docs.vscentrum.be/antwerp/tier2_hardware/leibniz_hardware.html'
            max_memory = 240.GB // 256 (total) - 16 GB (buffer)
            max_cpus = 28
            max_time = 3.day
        }
        process {
            executor = 'slurm'
            queue = 'broadwell_256'
        }
    }
    broadwell_256_local {
        params {
            config_profile_description = 'Broadwell_256 local profile for use on a single node of the Leibniz cluster of the CalcUA VSC HPC.'
            config_profile_contact = 'pmoris@itg.be (GitHub: @pmoris)'
            config_profile_url = 'https://docs.vscentrum.be/antwerp/tier2_hardware/leibniz_hardware.html'
            max_memory = get_allocated_mem(240) // 256 (total) - 16 GB (buffer)
            max_cpus = get_allocated_cpus(28)
            max_time = 3.day
        }
        process {
            executor = 'local'
        }
        partition_checker("broadwell_256")
    }
    // Breniac (previously Hopper) partitions
    skylake_slurm {
        params {
            config_profile_description = 'Skylake Slurm profile for use on the Breniac (former Hopper) cluster of the CalcUA VSC HPC.'
            config_profile_contact = 'pmoris@itg.be (GitHub: @pmoris)'
            config_profile_url = 'https://www.uantwerpen.be/en/research-facilities/calcua/infrastructure/'
            max_memory = 176.GB // 192 GB (total) - 16 GB (buffer)
            max_cpus = 28
            max_time = 7.day
        }
        process {
            executor = 'slurm'
            queue = 'skylake'
        }
    }
    skylake_local {
        params {
            config_profile_description = 'Skylake local profile for use on a single node of the Breniac (former Hopper) cluster of the CalcUA VSC HPC.'
            config_profile_contact = 'pmoris@itg.be (GitHub: @pmoris)'
            config_profile_url = 'https://www.uantwerpen.be/en/research-facilities/calcua/infrastructure/'
            max_memory = get_allocated_mem(176) // 192 GB (total) - 16 GB (buffer)
            max_cpus = get_allocated_cpus(28)
            max_time = 7.day
        }
        process {
            executor = 'local'
        }
        partition_checker("skylake")
    }
}
 
// Define functions to fetch the available CPUs and memory of the current execution node.
// Only used when running one of the *_local partition profiles and allows the cpu
// and memory thresholds to be set dynamic based on the available hardware as reported
// by Slurm. Can be supplied with a default return value, which should be set to the
// recommended thresholds for the particular partition's node types.
def get_allocated_cpus(int node_max_cpu) {
    max_cpus = System.getenv("SLURM_CPUS_PER_TASK") ?: System.getenv("SLURM_JOB_CPUS_PER_NODE") ?: node_max_cpu
    return max_cpus.toInteger()
}
def get_allocated_mem(int node_max_mem) {
    def mem_per_cpu = System.getenv("SLURM_MEM_PER_CPU")
    def cpus_per_task = System.getenv("SLURM_CPUS_PER_TASK") ?: System.getenv("SLURM_JOB_CPUS_PER_NODE")
 
    if ( mem_per_cpu && cpus_per_task ) {
        node_max_mem = mem_per_cpu.toInteger() / 1000 * cpus_per_task.toInteger()
    }
 
    return "${node_max_mem}.GB"
}