azurebatch Configuration

To be used with the azurebatchdev profile by specifying the -profile azurebatchdev when running nf-core pipelines. Custom queue and storage need to be supplied with params.az_location, params.batch_name, params.storage_name, params.principal_id, params.principal_secret, params.tenant_id

Required Parameters

--storage_name

Name of Azure blob storage account.

--az_location

The Azure Batch region where the computation is executed in VMs. Default (westus2).

--principal_id

The service principal client ID.

--principal_secret

The service principal client secret.

--tenant_id

The Azure tenant ID.

-w

The Azure Blob container to be used as Nextflow work directory (-w az://work).

--autopoolmode

Whether to use Nextflow autopool mode which creates an autoscaling pool for running Nextflow jobs. Defaults to false.

--allowpoolcreation

Allow Nextflow to create a pool for running Nextflow jobs. Defaults to false.

--deletejobs

Allow Nextflow to delete pools after completion. Defaults to true.

--acr_registry

URL to Azure container registry for private docker images.

--acr_username

Username to access private Azure container registry.

--acr_password

Password to access private Azure container registry.

Azure Batch Setup

Please refer to the Nextflow documentation which describe how to setup the Azure Batch environment.

Config file

See config file on GitHub

azurebatchdev.config
//Nextflow config file for running on Azure batch
params {
  config_profile_description = 'Azure BATCH Dev Cloud Profile'
  config_profile_contact = 'Venkat Malladi (@vsmalladi)'; Abhinav Sharma (@abhi18av)'
  config_profile_url = 'https://azure.microsoft.com/services/batch/'
 
  // Active Directory
  principal_id      = null
  principal_secret  = null
  tenant_id         = null
 
  // Storage
  storage_name      = null
 
  // Batch
  az_location       = "westus2"
  batch_name        = null
 
  vm_type           = "Standard_D8s_v3"
  autopoolmode      = false
  allowpoolcreation = true
  deletejobs        = true
  deletepools       = false
 
  // ACR
  acr_registry      = null
  acr_username      = null
  acr_password      = null
 
}
 
process {
    executor = "azurebatch"
}
 
azure {
  process {
      queue = 'Standard_D2d_v4'
      withLabel:process_low   {queue = 'Standard_D4d_v4'}
      withLabel:process_medium {queue = 'Standard_D16d_v4'}
      withLabel:process_high {queue = 'Standard_D32d_v4'}
      withLabel:process_high_memory {queue = 'Standard_D48d_v4'}
  }
  activeDirectory {
    servicePrincipalId = params.principal_id
    servicePrincipalSecret = params.principal_secret
    tenantId = params.tenant_id
  }
  storage {
      accountName = params.storage_name
  }
  batch {
      location                = params.az_location
      accountName             = params.batch_name
      tokenDuration           = "24h"
      autoPoolMode            = params.autopoolmode
      allowPoolCreation       = params.allowpoolcreation
      deleteJobsOnCompletion  = params.deletejobs
      deletePoolsOnCompletion = params.deletepools
      pools {
        Standard_D2d_v4 {
            autoScale = true
            vmType = 'Standard_D2d_v4'
            vmCount = 2
            maxVmCount = 20
            scaleFormula = '''
                $TargetLowPriorityNodes = 1;
                $TargetDedicatedNodes   = 0;
                $NodeDeallocationOption = taskcompletion;
            '''
        }
        Standard_D4d_v4 {
            autoScale = true
            vmType = 'Standard_D4d_v4'
            vmCount = 2
            maxVmCount = 20
            scaleFormula = '''
                $TargetLowPriorityNodes = 1;
                $TargetDedicatedNodes   = 0;
                $NodeDeallocationOption = taskcompletion;
            '''
        }
        Standard_D16d_v4 {
            autoScale = true
            vmType = 'Standard_D16d_v4'
            vmCount = 2
            maxVmCount = 20
            scaleFormula = '''
                $TargetLowPriorityNodes = 1;
                $TargetDedicatedNodes   = 0;
                $NodeDeallocationOption = taskcompletion;
            '''
        }
        Standard_D32d_v4 {
            autoScale = true
            vmType = 'Standard_D32d_v4'
            vmCount = 2
            maxVmCount = 20
            scaleFormula = '''
                $TargetLowPriorityNodes = 1;
                $TargetDedicatedNodes   = 0;
                $NodeDeallocationOption = taskcompletion;
            '''
        }
        Standard_D48d_v4 {
            autoScale = true
            vmType = 'Standard_D48d_v4'
            vmCount = 2
            maxVmCount = 10
            scaleFormula = '''
                $TargetLowPriorityNodes = 1;
                $TargetDedicatedNodes   = 0;
                $NodeDeallocationOption = taskcompletion;
            '''
        }
      }
  }
  registry {
      server   = params.acr_registry
      userName = params.acr_username
      password = params.acr_password
  }
}
azurebatchdev.config
//Nextflow config file for running on Azure batch
params {
  config_profile_description = 'Azure BATCH Dev Cloud Profile'
  config_profile_contact = 'Venkat Malladi (@vsmalladi)'; Abhinav Sharma (@abhi18av)'
  config_profile_url = 'https://azure.microsoft.com/services/batch/'
 
  // Active Directory
  principal_id      = null
  principal_secret  = null
  tenant_id         = null
 
  // Storage
  storage_name      = null
 
  // Batch
  az_location       = "westus2"
  batch_name        = null
 
  vm_type           = "Standard_D8s_v3"
  autopoolmode      = false
  allowpoolcreation = true
  deletejobs        = true
  deletepools       = false
 
  // ACR
  acr_registry      = null
  acr_username      = null
  acr_password      = null
 
}
 
process {
    executor = "azurebatch"
}
 
azure {
  process {
      queue = 'Standard_D2d_v4'
      withLabel:process_low   {queue = 'Standard_D4d_v4'}
      withLabel:process_medium {queue = 'Standard_D16d_v4'}
      withLabel:process_high {queue = 'Standard_D32d_v4'}
      withLabel:process_high_memory {queue = 'Standard_D48d_v4'}
  }
  activeDirectory {
    servicePrincipalId = params.principal_id
    servicePrincipalSecret = params.principal_secret
    tenantId = params.tenant_id
  }
  storage {
      accountName = params.storage_name
  }
  batch {
      location                = params.az_location
      accountName             = params.batch_name
      tokenDuration           = "24h"
      autoPoolMode            = params.autopoolmode
      allowPoolCreation       = params.allowpoolcreation
      deleteJobsOnCompletion  = params.deletejobs
      deletePoolsOnCompletion = params.deletepools
      pools {
        Standard_D2d_v4 {
            autoScale = true
            vmType = 'Standard_D2d_v4'
            vmCount = 2
            maxVmCount = 20
            scaleFormula = '''
                $TargetLowPriorityNodes = 1;
                $TargetDedicatedNodes   = 0;
                $NodeDeallocationOption = taskcompletion;
            '''
        }
        Standard_D4d_v4 {
            autoScale = true
            vmType = 'Standard_D4d_v4'
            vmCount = 2
            maxVmCount = 20
            scaleFormula = '''
                $TargetLowPriorityNodes = 1;
                $TargetDedicatedNodes   = 0;
                $NodeDeallocationOption = taskcompletion;
            '''
        }
        Standard_D16d_v4 {
            autoScale = true
            vmType = 'Standard_D16d_v4'
            vmCount = 2
            maxVmCount = 20
            scaleFormula = '''
                $TargetLowPriorityNodes = 1;
                $TargetDedicatedNodes   = 0;
                $NodeDeallocationOption = taskcompletion;
            '''
        }
        Standard_D32d_v4 {
            autoScale = true
            vmType = 'Standard_D32d_v4'
            vmCount = 2
            maxVmCount = 20
            scaleFormula = '''
                $TargetLowPriorityNodes = 1;
                $TargetDedicatedNodes   = 0;
                $NodeDeallocationOption = taskcompletion;
            '''
        }
        Standard_D48d_v4 {
            autoScale = true
            vmType = 'Standard_D48d_v4'
            vmCount = 2
            maxVmCount = 10
            scaleFormula = '''
                $TargetLowPriorityNodes = 1;
                $TargetDedicatedNodes   = 0;
                $NodeDeallocationOption = taskcompletion;
            '''
        }
      }
  }
  registry {
      server   = params.acr_registry
      userName = params.acr_username
      password = params.acr_password
  }
}
executor
azurebatch
contact
vsmalladi
homepage
https://azure.microsoft.com/services/batch/
get in touch
Ask a question on Slack Open an issue on GitHub