class oci::compute(
  $region_name              = 'RegionOne',
  $openstack_release        = undef,
  $cluster_name             = undef,
  $machine_hostname         = undef,
  $machine_ip               = undef,
  $bridge_mapping_list      = undef,
  $external_network_list    = undef,
  $first_master             = undef,
  $first_master_ip          = undef,
  $cluster_domain           = undef,
  $vmnet_ip                 = undef,

  $extswift_use_external            = false,
  $extswift_auth_url                = undef,
  $extswift_region                  = 'RegionOne',
  $extswift_proxy_url               = undef,
  $extswift_project_name            = undef,
  $extswift_project_domain_name     = 'Default',
  $extswift_user_name               = undef,
  $extswift_user_domain_name        = 'Default',
  $extswift_password                = undef,
  $cinder_backup_use_external_swift = no,

  $messaging_nodes_for_notifs = false,
  $all_rabbits                = [],
  $all_rabbits_ips            = [],


  $neutron_global_physnet_mtu = 1500,
  $neutron_path_mtu         = 1500,

  $all_masters              = undef,
  $all_masters_ip           = undef,
  $vip_hostname             = undef,
  $vip_ipaddr               = undef,
  $vip_netmask              = undef,
  $self_signed_api_cert     = true,
  $sql_vip_ip               = undef,
  $network_ipaddr           = undef,
  $network_cidr             = undef,
  $use_ssl                  = true,

  $pass_haproxy_stats       = undef,

  $pass_nova_messaging      = undef,
  $pass_nova_authtoken      = undef,
  $pass_nova_ssh_pub        = undef,
  $pass_nova_ssh_priv       = undef,
  $pass_glance_authtoken    = undef,
  $pass_neutron_authtoken   = undef,
  $pass_metadata_proxy_shared_secret = undef,
  $pass_neutron_messaging   = undef,
  $pass_neutron_vrrpauth    = undef,
  $pass_placement_authtoken = undef,

  $pass_manila_db           = undef,
  $pass_manila_messaging    = undef,
  $pass_manila_authtoken    = undef,

  $has_subrole_ceilometer   = false,
  $pass_ceilometer_messaging = undef,
  $pass_ceilometer_authtoken = undef,
  $pass_ceilometer_telemetry = undef,

  $pass_cinder_db           = undef,
  $pass_cinder_authtoken    = undef,
  $pass_cinder_messaging    = undef,

  $cluster_has_osds         = undef,
  $use_ceph_if_available    = false,
  $ceph_fsid                = undef,
  $ceph_libvirtuuid         = undef,
  $ceph_bootstrap_osd_key   = undef,
  $ceph_admin_key           = undef,
  $ceph_openstack_key       = undef,
  $ceph_mon_key             = undef,
  $ceph_mon_initial_members = undef,
  $ceph_mon_host            = undef,

  $disable_notifications    = false,
  $enable_monitoring_graphs = false,
  $monitoring_graphite_host = undef,
  $monitoring_graphite_port = '2003',

  $use_gpu                  = false,
  $gpu_name                 = undef,
  $gpu_vendor_id            = undef,
  $gpu_product_id           = undef,
  $gpu_device_type          = undef,
  $vfio_ids                 = undef,

  $has_subrole_dhcp_agent   = true,

  $nested_virt              = false,

  $cpu_mode                 = 'host-passthrough',
  $cpu_model                = undef,
  $cpu_model_extra_flags    = undef,

  $cpu_allocation_ratio     = undef,
  $ram_allocation_ratio     = undef,
  $disk_allocation_ratio    = undef,

  $bgp_to_the_host          = false,
  $dhcp_domain              = '',
  $backup_backend           = 'ceph',
  $has_subrole_designate    = false,
  $neutron_dns_domain       = 'example.com.',
  $admin_email_address      = 'admin@example.com',

  $use_dynamic_routing      = false,

  $neutron_install_dragent  = false,

  $neutron_use_dvr          = true,

  $nova_reserved_host_memory_mb = 16384,
  $nova_reserved_host_disk_mb   = 10240,

  $authorized_keys_hash     = [],

  # This one is unused, because we use the agregate of
  # network + bridge instead. However, it's transmitted
  # by the ENC because it is a machine variable, so this
  # variable must be present.
  $neutron_external_network_name = undef,

  $compute_is_cephosd       = false,

  $ceph_osd_initial_setup   = false,
  $block_devices            = undef,

  $has_subrole_manila       = false,
  $manila_share_install     = false,
  $install_designate        = false,
  $install_manila           = false,
  $install_telemetry        = true,

  $manila_network_cidr          = '10.240.0.0/12',
  $manila_network_division_mask = 28,

  $kernel_from_backports    = false,
  $brex_macaddr             = undef,
  $nested_openstack         = false,

  $neutron_vxlan_vni_min     = '1000',
  $neutron_vxlan_vni_max     = '9999',
){

  # Tweak performances for KVM host
  # class { '::oci::tuned': profile => 'virtual-host' }

  $packages = ['numactl','numad']
  package { $packages:
    ensure => installed,
  }->
  service { 'numad':
    ensure => 'stopped',
  }

  if $nested_openstack {
    $cpu_mode_real = 'none'
    $cpu_model_real = undef
  }else{
    $cpu_mode_real = $cpu_mode
    if $cpu_mode == 'host-passthrough'{
      $cpu_model_real = undef
    }else{
      $cpu_model_real = $cpu_model
    }
  }

  if $messaging_nodes_for_notifs {
    $notif_rabbit_servers = $all_rabbits
  }else{
    $notif_rabbit_servers = $all_masters
  }

  if $use_ssl {
    $proto = 'https'
    $messaging_default_port = '5671'
    $messaging_notify_port = '5671'
    $api_port = 443
  } else {
    $proto = 'http'
    $messaging_default_port = '5672'
    $messaging_notify_port = '5672'
    $api_port = 80
  }
  $messaging_default_proto = 'rabbit'
  $messaging_notify_proto = 'rabbit'

  $base_url = "${proto}://${vip_hostname}"
  $keystone_auth_uri  = "${base_url}:${api_port}/identity"
  $keystone_admin_uri = "${base_url}:${api_port}/identity"

  if $self_signed_api_cert {
    $api_endpoint_ca_file = '/etc/ssl/certs/oci-pki-oci-ca-chain.pem'
  }else{
    $api_endpoint_ca_file = ''
  }
  $oci_pki_root_ca_file = '/etc/ssl/certs/oci-pki-oci-ca-chain.pem'

  class { '::oci::puppet_oci_ca_cert':
    self_signed_api_cert => $self_signed_api_cert,
  }

  if $facts['os']['lsb'] != undef{
    $mycodename = $facts['os']['lsb']['distcodename']
  }else{
    $mycodename = $facts['os']['distro']['codename']
  }
  if $mycodename != 'stretch'{
    $libvirt_security_driver_content = 'security_driver = "apparmor"'
  }else{
    $libvirt_security_driver_content = '#security_driver = "apparmor"'
  }

  if $disable_notifications {
    $notification_driver = 'noop'
  }else{
    $notification_driver = 'messagingv2'
  }
  ############################################################
  ### HAProxy (used for metadata server proxying and such) ###
  ############################################################
  if $neutron_use_dvr {
    class { 'haproxy':
      global_options   => {
        'log'      => '/dev/log local0',
        'maxconn'  => '256',
        'user'     => 'haproxy',
        'group'    => 'haproxy',
        'daemon'   => '',
        'nbthread' => '4',
      },
      defaults_options => {
        'mode'    => 'http',
        'option'  => [
          'httplog',
        ],
      },
      merge_options => true,
    }->
    haproxy::listen { 'hapstats':
      section_name => 'statshttp',
      bind         => { "${machine_ip}:8088" => 'user root'},
      mode         => 'http',
      options      => {
          'stats' => [ 'uri /', "auth admin:${pass_haproxy_stats}", 'refresh 15', 'admin if TRUE', 'realm Haproxy Statistics', ],
        },
    }->
    haproxy::listen { 'haphealthcheck':
      section_name => 'healthcheck',
      bind         => { "${machine_ip}:8081" => 'user root'},
      mode         => 'health',
      options => [
         { 'option' => 'httpchk' },
         { 'option' => 'http-keep-alive' },

      ]
    }

    $haproxy_options_for_metadata_proxy = [
      { 'use_backend' => 'metadatabe' },
    ]
    haproxy::frontend { 'openstackfe':
      mode      => 'http',
      bind      => { "127.0.0.1:8775" => [] },
      options   => $haproxy_options_for_metadata_proxy,
    }
    haproxy::backend { 'metadatabe':
      options => [
         { 'option' => 'forwardfor' },
         { 'option' => 'httpchk GET /healthcheck' },
         { 'mode' => 'http' },
         { 'balance' => 'roundrobin' },
      ],
    }
    if $openstack_release == 'rocky'{
      $metadatabm_options = 'check'
    }else{
      $metadatabm_options = 'check check-ssl ssl verify required ca-file /etc/ssl/certs/oci-pki-oci-ca-chain.pem'
    }
    haproxy::balancermember { 'metadatabm':
      listening_service => 'metadatabe',
      ipaddresses       => $all_masters_ip,
      server_names      => $all_masters,
      ports             => 8775,
      options           => $metadatabm_options
    }
  } else {
    package { 'remove-haproxy-in-non-dvr':
      name   => 'haproxy',
      ensure => purged,
    }
  }

  ############
  ### Nova ###
  ############
  if $disable_notifications {
    $nova_notif_transport_url = ''
  } else {
    $nova_notif_transport_url = os_transport_url({
                                  'transport' => $messaging_notify_proto,
                                  'hosts'     => fqdn_rotate($notif_rabbit_servers),
                                  'port'      => $messaging_notify_port,
                                  'username'  => 'nova',
                                  'password'  => $pass_nova_messaging,
                                })
  }

  # This is needed for Buster, so live migration can work
  package { 'apparmor':
    ensure => present,
    before => Class['::nova'],
  }

  if ($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri') {
    class { '::nova':
      default_transport_url      => os_transport_url({
        'transport' => $messaging_default_proto,
        'hosts'     => fqdn_rotate($all_masters),
        'port'      => $messaging_default_port,
        'username'  => 'nova',
        'password'  => $pass_nova_messaging,
      }),
      notification_transport_url => $nova_notif_transport_url,
      database_connection    => '',
      rabbit_use_ssl         => $use_ssl,
      rabbit_ha_queues       => true,
      kombu_ssl_ca_certs     => $oci_pki_root_ca_file,
      amqp_sasl_mechanisms   => 'PLAIN',
      glance_api_servers     => "${base_url}/image",
      notification_driver    => $notification_driver,
      notify_on_state_change => 'vm_and_task_state',
      nova_public_key        => { type => 'ssh-rsa', key => $pass_nova_ssh_pub },
      nova_private_key       => { type => 'ssh-rsa', key => base64('decode', $pass_nova_ssh_priv) },
      cpu_allocation_ratio   => $cpu_allocation_ratio,
      ram_allocation_ratio   => $ram_allocation_ratio,
      disk_allocation_ratio  => $disk_allocation_ratio,
    }
  }else{
    if $openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena'{
      class { '::nova':
        default_transport_url      => os_transport_url({
          'transport' => $messaging_default_proto,
          'hosts'     => fqdn_rotate($all_masters),
          'port'      => $messaging_default_port,
          'username'  => 'nova',
          'password'  => $pass_nova_messaging,
        }),
        notification_transport_url => $nova_notif_transport_url,
        database_connection    => '',
        rabbit_use_ssl         => $use_ssl,
        rabbit_ha_queues       => true,
        kombu_ssl_ca_certs     => $oci_pki_root_ca_file,
        amqp_sasl_mechanisms   => 'PLAIN',
        notification_driver    => $notification_driver,
        notify_on_state_change => 'vm_and_task_state',
        nova_public_key        => { type => 'ssh-rsa', key => $pass_nova_ssh_pub },
        nova_private_key       => { type => 'ssh-rsa', key => base64('decode', $pass_nova_ssh_priv) },
        cpu_allocation_ratio   => $cpu_allocation_ratio,
        ram_allocation_ratio   => $ram_allocation_ratio,
        disk_allocation_ratio  => $disk_allocation_ratio,
        glance_endpoint_override      => "${base_url}/image",
      }
    # Yoga and up
    }else{
# We can't call the class like this anymore, because
# the class checks for a patern.
#      class { '::nova::db':
#        database_connection    => '',
#      }
      class { '::nova':
        default_transport_url      => os_transport_url({
          'transport' => $messaging_default_proto,
          'hosts'     => fqdn_rotate($all_masters),
          'port'      => $messaging_default_port,
          'username'  => 'nova',
          'password'  => $pass_nova_messaging,
        }),
        notification_transport_url => $nova_notif_transport_url,
        rabbit_use_ssl         => $use_ssl,
        rabbit_ha_queues       => true,
        kombu_ssl_ca_certs     => $oci_pki_root_ca_file,
        amqp_sasl_mechanisms   => 'PLAIN',
        notification_driver    => $notification_driver,
        notify_on_state_change => 'vm_and_task_state',
        nova_public_key        => { type => 'ssh-rsa', key => $pass_nova_ssh_pub },
        nova_private_key       => { type => 'ssh-rsa', key => base64('decode', $pass_nova_ssh_priv) },
        cpu_allocation_ratio   => $cpu_allocation_ratio,
        ram_allocation_ratio   => $ram_allocation_ratio,
        disk_allocation_ratio  => $disk_allocation_ratio,
      }
    }
    nova_config {
      'glance/api_servers': value => "${base_url}/image";
      'glance/region_name': value => $region_name;
    }
  }

  class { '::nova::keystone::authtoken':
    password             => $pass_nova_authtoken,
    auth_url             => $keystone_admin_uri,
    www_authenticate_uri => $keystone_auth_uri,
    cafile               => $api_endpoint_ca_file,
    region_name          => $region_name,
  }
  if $openstack_release != 'rocky' and $openstack_release != 'stein' {
    if $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena' or $openstack_release == 'yoga'{
      class { '::nova::cinder':
        password    => $pass_cinder_authtoken,
        region_name => $region_name,
        auth_url    => $keystone_admin_uri,
      }
    # Zed and up
    }else{
      class { '::nova::cinder':
        password       => $pass_cinder_authtoken,
        os_region_name => $region_name,
        auth_url       => $keystone_admin_uri,
      }
    }
  }


  # Needed, so puppet-openstack can restart virtlogd and libvirtd after configuration
  # so it listen to TCP before nova-compute starts.
  include ::nova::compute::libvirt::services
  Service['virtlogd'] -> Service['libvirtd']

  if ($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri') {
    # This enables live migrations over libvirt's TLS support
    class { '::nova::migration::libvirt':
      transport                      => 'tls',
      live_migration_inbound_addr    => $machine_hostname,
      live_migration_tunnelled       => false,
      live_migration_with_native_tls => true,
      ca_file                        => '/etc/pki/CA/cacert.pem',
    }
  }else{
    # This enables live migrations over libvirt's TLS support
    class { '::nova::migration::libvirt':
      transport                      => 'tls',
      live_migration_inbound_addr    => $machine_hostname,
      live_migration_tunnelled       => false,
      live_migration_with_native_tls => true,
      ca_file                        => '/etc/pki/CA/cacert.pem',
      modular_libvirt                => true,
    }
  }

  if $nested_openstack {
    $virt_type = 'qemu'
  }else{
    $virt_type = 'kvm'
  }

  if ($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri') {
    class { '::nova::compute::libvirt':
      disk_cachemodes                            => $disk_cachemodes,
      libvirt_virt_type                          => $virt_type,
      libvirt_cpu_mode                           => $cpu_mode_real,
      libvirt_cpu_model                          => $cpu_model_real,
      libvirt_cpu_model_extra_flags              => $cpu_model_extra_flags,
      migration_support                          => true,
      # False is needed because of the above include ::nova::compute::libvirt::services
      manage_libvirt_services                    => false,
      preallocate_images                         => 'space',
      # This is one week retention.
      remove_unused_original_minimum_age_seconds => '604800',
      vncserver_listen                           => '0.0.0.0',
      libvirt_hw_disk_discard                    => 'unmap',
    }
  }else{
    if ($openstack_release == 'victoria' or $openstack_release == 'wallaby') {
      class { '::nova::compute::libvirt':
        disk_cachemodes                            => $disk_cachemodes,
        virt_type                                  => $virt_type,
        cpu_mode                                   => $cpu_mode_real,
        cpu_models                                 => [ $cpu_model_real, ],
        cpu_model_extra_flags                      => $cpu_model_extra_flags,
        migration_support                          => true,
        # False is needed because of the above include ::nova::compute::libvirt::services
        manage_libvirt_services                    => false,
        preallocate_images                         => 'space',
        # This is one week retention.
        remove_unused_original_minimum_age_seconds => '604800',
        vncserver_listen                           => '0.0.0.0',
        libvirt_hw_disk_discard                    => 'unmap',
      }
    }else{
      class { '::nova::compute::libvirt':
        disk_cachemodes                            => $disk_cachemodes,
        virt_type                                  => $virt_type,
        cpu_mode                                   => $cpu_mode_real,
        cpu_models                                 => [ $cpu_model_real, ],
        cpu_model_extra_flags                      => $cpu_model_extra_flags,
        migration_support                          => true,
        # False is needed because of the above include ::nova::compute::libvirt::services
        manage_libvirt_services                    => false,
        preallocate_images                         => 'space',
        # This is one week retention.
        vncserver_listen                           => '0.0.0.0',
        hw_disk_discard                            => 'unmap',
      }
      class { '::nova::compute::image_cache':
        remove_unused_original_minimum_age_seconds => '604800',
      }
    }
  }
  create_resources( 'libvirtd_config', {
    'key_file'=>  { 'value' => '/etc/pki/libvirt/private/serverkey.pem', 'quote'  => true },
    'cert_file'=> { 'value' => '/etc/pki/libvirt/servercert.pem', 'quote'  => true },
  })

  # The +0 is there for converting the string to an int,
  # 1536 is for the PoC, 8192 is for real deployments where we do expect
  # hosts to hold more than 16GB of RAM.
  if (($::memorysize_mb + 0) < 16000) {
    $reserved_host_memory_mb = 4096
  }else{
    $reserved_host_memory_mb = $nova_reserved_host_memory_mb
  }

  nova_config {
    'placement/cafile':                 value => $api_endpoint_ca_file;
    'placement/auth_type':              value => 'password';
    'placement/auth_url':               value => $keystone_admin_uri;
    'placement/password':               value => $pass_placement_authtoken, secret => true;
    'placement/project_domain_name':    value => 'Default';
    'placement/project_name':           value => 'services';
    'placement/user_domain_name':       value => 'Default';
    'placement/username':               value => 'placement';
    'placement/region_name':            value => $region_name;
    'glance/cafile':                    value => $api_endpoint_ca_file;
    'keystone/cafile':                  value => $api_endpoint_ca_file;
    'neutron/cafile':                   value => $api_endpoint_ca_file;
    'cinder/cafile':                    value => $api_endpoint_ca_file;
    'service_user/cafile':              value => $api_endpoint_ca_file;
  }
  if ($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria') {
    nova_config {
      'DEFAULT/use_cow_images':           value => 'False';
      'DEFAULT/default_ephemeral_format': value => 'ext4';
    }
  }

  if ($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri') {
    nova_config {
      'DEFAULT/reserved_host_disk_mb':    value => $nova_reserved_host_disk_mb;
    }
  }

  if ($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri') {
    if $openstack_release != 'rocky' and $openstack_release != 'stein'{
      class { '::nova::compute::image_cache':
        manager_interval => '-1';
      }

    }
  }
  if $openstack_release != 'rocky' and $openstack_release != 'stein'{
    # Every 6 hours, clean the _base image folder of Nova, otherwise it
    # may become quickly super big.
    cron { 'oci-compute-node-image-cache-cleaner':
      command => '/usr/bin/flock -w 0 /var/lock/oci-compute-node-image-cache-cleaner.lock /usr/bin/oci-compute-node-image-cache-cleaner >/dev/null 2>&1',
      user    => 'nova',
      hour    => [0, 6, 12, 18],
      require => Anchor['nova::install::end'],
    }
  }

  # Note: this *must* be set *before* calling ::nova::compute, as ::nova::compute is
  # doing an include ::nova::pci.
  if $use_gpu {

    $aliases_array = $gpu_name.map |Integer $i, String $n| {
      {"vendor_id" => $gpu_vendor_id[$i], "product_id" => $gpu_product_id[$i], "name" => $gpu_name[$i], "device_type" => $gpu_device_type}
    }
    $passthrough_array = $gpu_name.map |Integer $i, String $n| {
      {"vendor_id" => $gpu_vendor_id[$i], "product_id" => $gpu_product_id[$i]}
    }
    #notify { "GPU aliases array: ${aliases_array}": withpath => true }
    #notify { "GPU passthrough array: ${passthrough_array}": withpath => true }

    class { '::nova::pci':
      aliases => $aliases_array,
    }
    class {'::nova::compute::pci':
      passthrough => $passthrough_array,
    }
    file { '/etc/modprobe.d/blacklist-nvidia.conf':
      ensure                  => present,
      owner                   => 'root',
      content                 => 'blacklist nouveau
blacklist nvidiafb
blacklist snd_hda_intel
',
      selinux_ignore_defaults => true,
      mode                    => '0644',
    }
    file { '/etc/modprobe.d/vfio.conf':
      ensure                  => present,
      owner                   => 'root',
      content                 => "options vfio-pci ids=${vfio_ids}",
      selinux_ignore_defaults => true,
      mode                    => '0644',
    }
  }else{
    file { '/etc/modprobe.d/blacklist-nvidia.conf':
      ensure                  => absent,
    }
    file { '/etc/modprobe.d/vfio.conf':
      ensure                  => absent,
    }
  }
  # Needed to expose physical devices to VMs
  file { '/etc/modules-load.d/vfio.conf':
    ensure                  => present,
    owner                   => 'root',
    content                 => "# Managed by puppet: do not touch

vfio-pci
",
    selinux_ignore_defaults => true,
    mode                    => '0644',
  }

  if $openstack_release == 'rocky'{
    class { '::nova::compute':
      vnc_enabled                      => true,
      vncproxy_host                    => "${vip_hostname}",
      vncproxy_protocol                => 'https',
      vncproxy_port                    => '443',
      vncproxy_path                    => '/novnc/vnc_auto.html',
      vncserver_proxyclient_address    => $machine_ip,
      instance_usage_audit             => true,
      instance_usage_audit_period      => 'hour',
      barbican_auth_endpoint           => "${base_url}/identity/v3",
      barbican_endpoint                => "${base_url}/keymanager",
      keymgr_backend                   => 'barbican',
      resume_guests_state_on_host_boot => true,
      allow_resize_to_same_host        => true,
      force_config_drive               => true,
      reserved_host_memory             => $reserved_host_memory_mb,
#     resize_confirm_window            => '60',
    }
  } else {
    if ($openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri') {
      class { '::nova::compute':
        vnc_enabled                      => true,
        vncproxy_host                    => "${vip_hostname}",
        vncproxy_protocol                => 'https',
        vncproxy_port                    => '443',
        vncproxy_path                    => '/novnc/vnc_auto.html',
        vncserver_proxyclient_address    => $machine_ip,
        instance_usage_audit             => true,
        instance_usage_audit_period      => 'hour',
        barbican_auth_endpoint           => "${base_url}/identity/v3",
        barbican_endpoint                => "${base_url}/keymanager",
        keymgr_backend                   => 'barbican',
        resume_guests_state_on_host_boot => true,
        allow_resize_to_same_host        => true,
        force_config_drive               => true,
        reserved_host_memory             => $reserved_host_memory_mb,
  #     resize_confirm_window            => '60',
      }
    }else{
      if $openstack_release == 'victoria' {
        class { '::nova::compute':
          vnc_enabled                      => true,
          vncproxy_host                    => "${vip_hostname}",
          vncproxy_protocol                => 'https',
          vncproxy_port                    => '443',
          vncproxy_path                    => '/novnc/vnc_auto.html',
          vncserver_proxyclient_address    => $machine_ip,
          instance_usage_audit             => true,
          instance_usage_audit_period      => 'hour',
          barbican_auth_endpoint           => "${base_url}/identity/v3",
          barbican_endpoint                => "${base_url}/keymanager",
          keymgr_backend                   => 'barbican',
          resume_guests_state_on_host_boot => true,
          allow_resize_to_same_host        => true,
          force_config_drive               => true,
          reserved_host_memory             => $reserved_host_memory_mb,
          reserved_host_disk               => $nova_reserved_host_disk_mb,
        }
      }elsif $openstack_release == 'wallaby'{
        class { '::nova::compute':
          vnc_enabled                      => true,
          vncproxy_host                    => "${vip_hostname}",
          vncproxy_protocol                => 'https',
          vncproxy_port                    => '443',
          vncproxy_path                    => '/novnc/vnc_auto.html',
          vncserver_proxyclient_address    => $machine_ip,
          instance_usage_audit             => true,
          instance_usage_audit_period      => 'hour',
          barbican_auth_endpoint           => "${base_url}/identity/v3",
          barbican_endpoint                => "${base_url}/keymanager",
          keymgr_backend                   => 'barbican',
          resume_guests_state_on_host_boot => true,
          force_config_drive               => true,
          reserved_host_memory             => $reserved_host_memory_mb,
          reserved_host_disk               => $nova_reserved_host_disk_mb,
          use_cow_images                   => 'False',
          default_ephemeral_format         => 'ext4',
        }
      }else{
        class { '::nova::key_manager::barbican':
          barbican_endpoint => "${base_url}/keymanager",
          auth_endpoint     => "${base_url}/identity/v3",
        }
        class { '::nova::compute':
          vnc_enabled                      => true,
          vncproxy_host                    => "${vip_hostname}",
          vncproxy_protocol                => 'https',
          vncproxy_port                    => '443',
          vncproxy_path                    => '/novnc/vnc_auto.html',
          vncserver_proxyclient_address    => $machine_ip,
          instance_usage_audit             => true,
          instance_usage_audit_period      => 'hour',
          resume_guests_state_on_host_boot => true,
          force_config_drive               => true,
          reserved_host_memory             => $reserved_host_memory_mb,
          reserved_host_disk               => $nova_reserved_host_disk_mb,
          use_cow_images                   => 'False',
          default_ephemeral_format         => 'ext4',
        }
      }
    }
    class { '::nova::logging':
      debug => true,
    }
  }

  if $use_ceph_if_available or $compute_is_cephosd {
    $disk_cachemodes = ['"network=writeback"']
  }



  file_line { 'parallel-shutdown-of-vms':
    path    => '/etc/default/libvirt-guests',
    match   => '.*PARALLEL_SHUTDOWN.*=.*',
    line    => 'PARALLEL_SHUTDOWN=8',
    require => Class['::nova::compute::libvirt'],
  }->
  file_line { 'shutdown-timeout-of-vms':
    path   => '/etc/default/libvirt-guests',
    match  => '.*SHUTDOWN_TIMEOUT.*=.*',
    line   => 'SHUTDOWN_TIMEOUT=120',
  }->
  file_line { 'start-delay-of-vms':
    path   => '/etc/default/libvirt-guests',
    match  => '.*START_DELAY.*=.*',
    line   => 'START_DELAY=4',
  }->
  file_line { 'use-apparmor-not-selinux':
    path   => '/etc/libvirt/qemu.conf',
    match  => '^#?security_driver.*=.*',
    line   => $libvirt_security_driver_content,
    notify => Service['libvirtd'],
  }->
  file_line { 'enable-vnc-tls':
    path   => '/etc/libvirt/qemu.conf',
    match  => '^#?vnc_tls[ \t]*=.*',
    line   => 'vnc_tls=1',
    notify => Service['libvirtd'],
  }->
  file_line { 'vnc-tls-x509-verify':
    path   => '/etc/libvirt/qemu.conf',
    match  => '^#?vnc_tls_x509_verify[ \t]*=.*',
    line   => 'vnc_tls_x509_verify=1',
    notify => Service['libvirtd'],
  }

  if $nested_virt{
    file { '/etc/modprobe.d/kvm.conf':
      ensure  => present,
      content => 'options kvm_intel nested=1',
    }
    file { '/etc/modprobe.d/kvm_amd.conf':
      ensure  => present,
      content => 'options kvm_amd nested=1',
    }
  }else{
    file { '/etc/modprobe.d/kvm.conf':
      ensure  => present,
      content => 'options kvm_intel nested=0',
    }
    file { '/etc/modprobe.d/kvm_amd.conf':
      ensure  => present,
      content => 'options kvm_intel nested=0',
    }
  }

  if $openstack_release == 'rocky'{
    class { '::nova::network::neutron':
      neutron_auth_url      => "${base_url}/identity/v3",
      neutron_url           => "${base_url}/network",
      neutron_password      => $pass_neutron_authtoken,
      default_floating_pool => 'public',
      dhcp_domain           => $dhcp_domain,
      neutron_region_name   => $region_name,
    }
    nova_config {
      'neutron/endpoint_override': value => "${base_url}/network";
    }
  }else{
    if $openstack_release == 'stein' or $openstack_release == 'train'{
      class { '::nova::network::neutron':
        neutron_auth_url          => "${base_url}/identity/v3",
        neutron_url               => "${base_url}/network",
        neutron_password          => $pass_neutron_authtoken,
        default_floating_pool     => 'public',
        dhcp_domain               => $dhcp_domain,
        neutron_endpoint_override => "${base_url}/network",
        neutron_region_name       => $region_name,
      }
    }else{
      class { '::nova::network::neutron':
        auth_url              => "${base_url}/identity/v3",
        password              => $pass_neutron_authtoken,
        default_floating_pool => 'public',
        endpoint_override     => "${base_url}/network",
        region_name           => $region_name,
      }
      class { '::nova::metadata':
        dhcp_domain                          => $dhcp_domain,
        neutron_metadata_proxy_shared_secret => $pass_metadata_proxy_shared_secret,
      }
    }
  }

  if $cluster_has_osds {
    class { 'ceph':
      fsid                => $ceph_fsid,
      ensure              => 'present',
      authentication_type => 'cephx',
      mon_initial_members => $ceph_mon_initial_members,
      mon_host            => $ceph_mon_host,
    }->
    ceph::key { 'client.admin':
      secret  => $ceph_admin_key,
      cap_mon => 'allow *',
      cap_osd => 'allow *',
      cap_mds => 'allow',
    }->
    ceph::key { 'client.openstack':
      secret  => $ceph_openstack_key,
      mode    => '0644',
      cap_mon => 'profile rbd',
      cap_osd => 'profile rbd pool=cinder, profile rbd pool=nova, profile rbd pool=glance, profile rbd pool=gnocchi, profile rbd pool=cinderback',
    }->
    ceph::key { 'client.bootstrap-osd':
      secret       => $ceph_bootstrap_osd_key,
      keyring_path => '/var/lib/ceph/bootstrap-osd/ceph.keyring',
      cap_mon      => 'allow profile bootstrap-osd',
    }->
    class { '::ceph::profile::params':
      fsid                => $ceph_fsid,
      manage_repo         => false,
      release             => 'nautilus',
      authentication_type => 'cephx',
      mon_host            => $ceph_mon_host,
      mon_initial_members => $ceph_mon_initial_members,
      client_keys         => {
        'client.admin' => {
          secret  => $ceph_admin_key,
          cap_mon => 'allow *',
          cap_osd => 'allow *',
          cap_mds => 'allow',
        },
        'client.openstack' => {
          secret  => $ceph_openstack_key,
          mode    => '0644',
          cap_mon => 'profile rbd',
          cap_osd => 'profile rbd pool=cinder, profile rbd pool=nova, profile rbd pool=glance, profile rbd pool=gnocchi, profile rbd pool=cinderback',
        },
        'client.bootstrap-osd' => {
          secret       => $ceph_bootstrap_osd_key,
          keyring_path => '/var/lib/ceph/bootstrap-osd/ceph.keyring',
          cap_mon      => 'allow profile bootstrap-osd',
        }
      }
    }

    if $compute_is_cephosd and $ceph_osd_initial_setup{
      $block_devices.each |Integer $index, String $value| {
        ceph::osd { "/dev/${value}":
          dmcrypt => false,
          require => Class['::ceph::profile::params'],
        }
      }
    }

    if $use_ceph_if_available or $compute_is_cephosd{
      class { '::nova::compute::rbd':
        libvirt_rbd_user        => 'openstack',
        libvirt_rbd_secret_uuid => $ceph_libvirtuuid,
        libvirt_rbd_secret_key  => $ceph_openstack_key,
        libvirt_images_rbd_pool => 'nova',
        rbd_keyring             => 'client.openstack',
        # ceph packaging is already managed by puppet-ceph
        manage_ceph_client      => false,
        require => Ceph::Key['client.openstack'],
      }
    }else{
      # If we have Ceph OSDs but we don't use /var/lib/nova/instances,
      # we still need the secret to be defined on libvirt: let's do it
      # without ::nova::compute::rbd, and by the way, let's disable
      # RBD on nova.conf, just in case someone changed his mind, and
      # disabled such config on the node.

      $rbd_keyring = 'client.openstack'
      $libvirt_rbd_secret_uuid = $ceph_libvirtuuid
      file { '/etc/nova/secret.xml':
        content => template('nova/secret.xml-compute.erb'),
        require => Ceph::Key['client.openstack'],
      }

      #Variable name shrunk in favor of removing
      #the more than 140 chars puppet-lint warning.
      #variable used in the get-or-set virsh secret
      #resource.
      $cm = '/usr/bin/virsh secret-define --file /etc/nova/secret.xml | /usr/bin/awk \'{print $2}\' | sed \'/^$/d\' > /etc/nova/virsh.secret'
      exec { 'get-or-set virsh secret':
        command => $cm,
        unless  => "/usr/bin/virsh secret-list | grep -i ${libvirt_rbd_secret_uuid}",
        require => File['/etc/nova/secret.xml'],
      }
      $cm2 = "/usr/bin/virsh secret-set-value --secret ${libvirt_rbd_secret_uuid} ${ceph_openstack_key}"
      exec { 'get-or-set virsh secret value':
        command => $cm2,
        unless  => "/usr/bin/virsh secret-get-value ${libvirt_rbd_secret_uuid} 2>/dev/null | grep -i ${ceph_openstack_key}",
        require => Exec['get-or-set virsh secret'],
      }
      Service<| title == 'libvirt' |> -> Exec['get-or-set virsh secret'] -> Exec['get-or-set virsh secret value']
      if $openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena'{
        nova_config {
          'libvirt/images_type':          ensure => absent;
        }
      }
      nova_config {
        'libvirt/images_rbd_pool':      ensure => absent;
        'libvirt/images_rbd_ceph_conf': ensure => absent;
        'libvirt/rbd_secret_uuid':      value => $ceph_libvirtuuid;
        'libvirt/rbd_user':             value => 'openstack';
      }
      # make sure ceph pool exists before running nova-compute
      # Note: cannot be done on the compute nodes, since we aren't doing pools here
      # but on the controller.
      #Exec['create-nova'] -> Service['nova-compute']
    }
  }

  ###############
  ### Neutron ###
  ###############
  if $disable_notifications {
    $neutron_notif_transport_url = ''
  } else {
    $neutron_notif_transport_url = os_transport_url({
                                     'transport' => $messaging_notify_proto,
                                     'hosts'     => fqdn_rotate($notif_rabbit_servers),
                                     'port'      => $messaging_notify_port,
                                     'username'  => 'neutron',
                                     'password'  => $pass_neutron_messaging,
                                   })
  }

  if $neutron_use_dvr {
    $l3_agent_mode              = 'dvr'
    $enable_distributed_routing = true
    $install_l3_agent           = true
  }else{
    $l3_agent_mode              = 'legacy'
    $enable_distributed_routing = false
    $install_l3_agent           = false
  }

  $bgp_dr_plugin = $use_dynamic_routing ? {
    true    => 'bgp',
    default => undef,
  }
  $plugins_list = delete_undef_values(['router', 'metering', 'qos', 'trunk', 'segments', $bgp_dr_plugin])
  if $neutron_install_dragent {
    class { '::neutron::agents::bgp_dragent':
      bgp_router_id => $machine_ip,
    }
  }else{
    package { 'neutron dynamic routing':
      name   => 'neutron-bgp-dragent',
      ensure => purged,
    }
    if $use_dynamic_routing {
      package { 'neutron-dynamic-routing-common':
        name   => 'neutron-dynamic-routing-common',
        ensure => present,
      }
    }else{
      package { 'python3-neutron-dynamic-routing':
        name   => 'python3-neutron-dynamic-routing',
        ensure => purged,
      }
    }
  }

  if $openstack_release == 'rocky'{
    class { '::neutron':
      debug                      => true,
      default_transport_url      => os_transport_url({
        'transport' => $messaging_default_proto,
        'hosts'     => fqdn_rotate($all_masters),
        'port'      => $messaging_default_port,
        'username'  => 'neutron',
        'password'  => $pass_neutron_messaging,
      }),
      notification_transport_url => $neutron_notif_transport_url,
      rabbit_use_ssl             => $use_ssl,
      rabbit_ha_queues           => true,
      kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
      amqp_sasl_mechanisms       => 'PLAIN',
      allow_overlapping_ips      => true,
      core_plugin                => 'ml2',
      service_plugins            => $plugins_list,
      bind_host                  => $machine_ip,
      notification_driver        => $notification_driver,
      global_physnet_mtu         => $neutron_global_physnet_mtu,
      dns_domain                 => $neutron_dns_domain,
    }
  }elsif $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena' or $openstack_release == 'yoga' {
    class { '::neutron':
      default_transport_url      => os_transport_url({
        'transport' => $messaging_default_proto,
        'hosts'     => fqdn_rotate($all_masters),
        'port'      => $messaging_default_port,
        'username'  => 'neutron',
        'password'  => $pass_neutron_messaging,
      }),
      notification_transport_url => $neutron_notif_transport_url,
      rabbit_use_ssl             => $use_ssl,
      rabbit_ha_queues           => true,
      kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
      amqp_sasl_mechanisms       => 'PLAIN',
      allow_overlapping_ips      => true,
      core_plugin                => 'ml2',
      service_plugins            => $plugins_list,
      bind_host                  => $machine_ip,
      notification_driver        => $notification_driver,
      global_physnet_mtu         => $neutron_global_physnet_mtu,
      dns_domain                 => $neutron_dns_domain,
    }
  }else{
    class { '::neutron':
      default_transport_url      => os_transport_url({
        'transport' => $messaging_default_proto,
        'hosts'     => fqdn_rotate($all_masters),
        'port'      => $messaging_default_port,
        'username'  => 'neutron',
        'password'  => $pass_neutron_messaging,
      }),
      notification_transport_url => $neutron_notif_transport_url,
      rabbit_use_ssl             => $use_ssl,
      rabbit_ha_queues           => true,
      kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
      amqp_sasl_mechanisms       => 'PLAIN',
      core_plugin                => 'ml2',
      service_plugins            => $plugins_list,
      bind_host                  => $machine_ip,
      notification_driver        => $notification_driver,
      global_physnet_mtu         => $neutron_global_physnet_mtu,
      dns_domain                 => $neutron_dns_domain,
    }
  }
  neutron_config {
    'nova/cafile':         value => $api_endpoint_ca_file;
    'database/connection': value => '';
  }
  if $openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri'{
    class { '::neutron::server::notifications':
      auth_url => $keystone_admin_uri,
      password => $pass_nova_authtoken,
    }
  # Victoria and up
  }else{
    class { 'neutron::server::notifications::nova':
      auth_url    => $keystone_admin_uri,
      password    => $pass_nova_authtoken,
      region_name => $region_name,
    }
  }
  class { '::neutron::client': }
  class { '::neutron::keystone::authtoken':
    password             => $pass_neutron_authtoken,
    auth_url             => $keystone_admin_uri,
    www_authenticate_uri => $keystone_auth_uri,
    cafile               => $api_endpoint_ca_file,
    region_name          => $region_name,
  }
  class { '::neutron::agents::ml2::ovs':
    local_ip                   => $vmnet_ip,
    tunnel_types               => ['vxlan'],
    bridge_uplinks             => ['eth0'],
    bridge_mappings            => $bridge_mapping_list,
    extensions                 => '',
    l2_population              => true,
    arp_responder              => true,
    firewall_driver            => 'iptables_hybrid',
    drop_flows_on_start        => false,
    enable_distributed_routing => $enable_distributed_routing,
    manage_vswitch             => false,
  }
  if $install_l3_agent {
    class { '::neutron::agents::l3':
      interface_driver      => 'openvswitch',
      debug                 => true,
      agent_mode            => $l3_agent_mode,
      ha_enabled            => false,
      extensions            => '',
      ha_vrrp_auth_type     => 'PASS',
      ha_vrrp_auth_password => $pass_neutron_vrrpauth,
    }
    neutron_l3_agent_config {
      'DEFAULT/external_network_bridge': value => '';
    }
  } else {
    package { 'neutron-l3-agent':
      ensure => absent,
    }
  }

  if $has_subrole_designate {
    $extension_drivers = 'port_security,qos,dns,dns_domain_ports'
  }else{
    $extension_drivers = 'port_security,qos'
  }

  if($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri'){
    class { '::neutron::plugins::ml2':
      type_drivers         => ['flat', 'vxlan', 'vlan', ],
      tenant_network_types => ['flat', 'vxlan', 'vlan', ],
      extension_drivers    => $extension_drivers,
      mechanism_drivers    => 'openvswitch,l2population',
      firewall_driver      => 'iptables_v2',
      flat_networks        => $external_network_list,
      network_vlan_ranges  => $external_network_list,
      vni_ranges           => "${neutron_vxlan_vni_min}:${neutron_vxlan_vni_max}",
      path_mtu             => $neutron_path_mtu,
    }
  }else{
    class { '::neutron::plugins::ml2':
      type_drivers         => ['flat', 'vxlan', 'vlan', ],
      tenant_network_types => ['flat', 'vxlan', 'vlan', ],
      extension_drivers    => $extension_drivers,
      mechanism_drivers    => 'openvswitch,l2population',
      flat_networks        => $external_network_list,
      network_vlan_ranges  => $external_network_list,
      vni_ranges           => "${neutron_vxlan_vni_min}:${neutron_vxlan_vni_max}",
      path_mtu             => $neutron_path_mtu,
    }
  }
  if $has_subrole_dhcp_agent {
    class { '::neutron::agents::dhcp':
      interface_driver         => 'openvswitch',
      debug                    => true,
      enable_isolated_metadata => true,
      enable_metadata_network  => true,
    }
  }else{
    package { 'neutron-dhcp-agent':
      ensure => absent,
    }
  }
  class { '::neutron::agents::metering':
    interface_driver => 'openvswitch',
    driver           => 'iptables',
    debug            => true,
  }
  if $neutron_use_dvr {
    class { '::neutron::agents::metadata':
      debug             => true,
      shared_secret     => $pass_metadata_proxy_shared_secret,
      metadata_workers  => 2,
      metadata_protocol => 'http',
      metadata_host     => 'localhost',
    }
  } else {
    package { 'remove-metadata-agent-in-non-dvr':
      name   => 'neutron-metadata-agent',
      ensure => purged,
    }
  }
  package { 'l2gw networking':
    name => 'python3-networking-l2gw',
    ensure => installed,
  }
  ##############
  ### Cinder ###
  ##############
  # If using Ceph, then we setup cinder-volume and cinder-backup over Ceph
  # on each and every compute nodes. The goal is to spread the load away
  # from the controller nodes.
  if $cluster_has_osds {
    include ::cinder::client

    if $disable_notifications {
      $cinder_notif_transport_url = ''
    } else {
      $cinder_notif_transport_url = os_transport_url({
                                      'transport' => $messaging_notify_proto,
                                      'hosts'     => fqdn_rotate($notif_rabbit_servers),
                                      'port'      => $messaging_notify_port,
                                      'username'  => 'cinder',
                                      'password'  => $pass_cinder_messaging,
                                    })
    }

    # Cinder main class (ie: cinder-common config)
    if $openstack_release == 'rocky'{
      class { '::cinder':
        debug                 => true,
        default_transport_url => os_transport_url({
          'transport' => $messaging_default_proto,
          'hosts'     => fqdn_rotate($all_masters),
          'port'      => $messaging_default_port,
          'username'  => 'cinder',
          'password'  => $pass_cinder_messaging,
        }),
        database_connection   => "mysql+pymysql://cinder:${pass_cinder_db}@${sql_vip_ip}/cinderdb?charset=utf8",
        rabbit_use_ssl        => $use_ssl,
        rabbit_ha_queues      => true,
        kombu_ssl_ca_certs    => $oci_pki_root_ca_file,
        amqp_sasl_mechanisms  => 'PLAIN',
      }
    }else{
      if($openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri'){
        class { '::cinder':
          default_transport_url => os_transport_url({
            'transport' => $messaging_default_proto,
            'hosts'     => fqdn_rotate($all_masters),
            'port'      => $messaging_default_port,
            'username'  => 'cinder',
            'password'  => $pass_cinder_messaging,
          }),
          database_connection   => "mysql+pymysql://cinder:${pass_cinder_db}@${sql_vip_ip}/cinderdb?charset=utf8",
          rabbit_use_ssl        => $use_ssl,
          rabbit_ha_queues      => true,
          kombu_ssl_ca_certs    => $oci_pki_root_ca_file,
          amqp_sasl_mechanisms  => 'PLAIN',
        }
        class { 'cinder::ceilometer':
          notification_transport_url => $cinder_notif_transport_url,
          notification_driver        => $notification_driver,
        }
      }else{
        if $openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena'{
          class { '::cinder':
            default_transport_url => os_transport_url({
              'transport' => $messaging_default_proto,
              'hosts'     => fqdn_rotate($all_masters),
              'port'      => $messaging_default_port,
              'username'  => 'cinder',
              'password'  => $pass_cinder_messaging,
            }),
            database_connection   => "mysql+pymysql://cinder:${pass_cinder_db}@${sql_vip_ip}/cinderdb?charset=utf8",
            rabbit_use_ssl        => $use_ssl,
            rabbit_ha_queues      => true,
            kombu_ssl_ca_certs    => $oci_pki_root_ca_file,
            amqp_sasl_mechanisms  => 'PLAIN',
            notification_transport_url => $cinder_notif_transport_url,
            notification_driver        => $notification_driver,
          }
        }else{
          class { '::cinder::db':
            database_connection              => "mysql+pymysql://cinder:${pass_cinder_db}@${sql_vip_ip}/cinderdb?charset=utf8",
            database_connection_recycle_time => 1800,
          }
          class { '::cinder':
            default_transport_url => os_transport_url({
              'transport' => $messaging_default_proto,
              'hosts'     => fqdn_rotate($all_masters),
              'port'      => $messaging_default_port,
              'username'  => 'cinder',
              'password'  => $pass_cinder_messaging,
            }),
            rabbit_use_ssl        => $use_ssl,
            rabbit_ha_queues      => true,
            kombu_ssl_ca_certs    => $oci_pki_root_ca_file,
            amqp_sasl_mechanisms  => 'PLAIN',
            notification_transport_url => $cinder_notif_transport_url,
            notification_driver        => $notification_driver,
          }
        }
      }
    }

    if $extswift_use_external and $cinder_backup_use_external_swift{
      $backup_backend_real         = 'cinder.backup.drivers.swift.SwiftBackupDriver'
      $backup_swift_auth_url       = $extswift_auth_url
      $backup_swift_url            = $extswift_proxy_url
      $backup_swift_user_domain    = $extswift_user_domain_name
      $backup_swift_project_domain = $extswift_project_domain_name
      $backup_swift_project        = $extswift_project_name
      $backup_swift_user           = $extswift_user_name
      $backup_swift_key            = $extswift_password
      $backup_swift_auth_type      = 'single_user'
    }else{
      $backup_swift_auth_url       = $keystone_auth_uri
      $backup_swift_url            = undef
      $backup_swift_user_domain    = undef
      $backup_swift_project_domain = undef
      $backup_swift_project        = undef
      $backup_swift_user           = undef
      $backup_swift_key            = undef
      $backup_swift_auth_type      = 'per_user'
      case $backup_backend {
        'ceph': {
          $backup_backend_real = 'cinder.backup.drivers.ceph.CephBackupDriver'
        }
        'swift': {
          $backup_backend_real = 'cinder.backup.drivers.swift.SwiftBackupDriver'
        }
        'none': {
          $backup_backend_real = ''
        }
      }
    }

    cinder_config {
      'service_user/cafile':                 value => $api_endpoint_ca_file;
      'DEFAULT/backup_driver':               value => $backup_backend_real;
      'DEFAULT/snapshot_clone_size':         value => '200';
      'DEFAULT/rbd_pool':                    value => 'cinder';
      'DEFAULT/rbd_ceph_conf':               value => '/etc/ceph/ceph.conf';
      'DEFAULT/rbd_secret_uuid':             value => $ceph_libvirtuuid;
      'DEFAULT/backup_swift_auth_url':       value => $backup_swift_auth_url;
      'DEFAULT/backup_swift_url':            value => $backup_swift_url;
      'DEFAULT/backup_swift_auth':           value => $backup_swift_auth_type;
      'DEFAULT/backup_swift_user_domain':    value => $backup_swift_user_domain;
      'DEFAULT/backup_swift_project_domain': value => $backup_swift_project_domain;
      'DEFAULT/backup_swift_project':        value => $backup_swift_project;
      'DEFAULT/backup_swift_user':           value => $backup_swift_user;
      'DEFAULT/backup_swift_key':            value => $backup_swift_key;
      'DEFAULT/backup_swift_auth_version':   value => '3';
      'DEFAULT/keystone_catalog_info':       value => 'identity:keystone:publicURL';
      'DEFAULT/backup_ceph_user':            value => 'openstack';
      'DEFAULT/backup_ceph_pool':            value => 'cinderback';
      'DEFAULT/backup_swift_ca_cert_file':   value => $api_endpoint_ca_file;
      'ssl/ca_file':                         value => $api_endpoint_ca_file;
    }

    # Configure the authtoken
    class { '::cinder::keystone::authtoken':
      password             => $pass_cinder_authtoken,
      auth_url             => $keystone_admin_uri,
      www_authenticate_uri => $keystone_auth_uri,
      memcached_servers    => $memcached_servers,
      cafile               => $api_endpoint_ca_file,
      region_name          => $region_name,
    }

    class { '::cinder::backends':
      enabled_backends => ['CEPH_1'],
    }

    cinder::backend::rbd { 'CEPH_1':
      rbd_user           => 'openstack',
      rbd_pool           => 'cinder',
      rbd_secret_uuid    => $ceph_libvirtuuid,
      manage_volume_type => true,
      backend_host       => $machine_hostname,
    }

    # Clear volumes on delete (for data security)
    class { '::cinder::volume':
      volume_clear => 'zero',
    }

    # A cinder-backup service on each volume nodes
    class { '::cinder::backup': }

    # Avoids Cinder to lookup for the catalogue
    class { '::cinder::glance':
      glance_api_servers => "${base_url}/image",
    }
    class { '::cinder::nova':
      password    => $pass_nova_authtoken,
      auth_url    => $keystone_admin_uri,
      cafile      => $api_endpoint_ca_file,
      region_name => $region_name,
      auth_type   => 'password',
    }
  }

  ##################
  ### Ceilometer ###
  ##################
  if $has_subrole_ceilometer and $install_telemetry{
    if $openstack_release == 'rocky'{
      class { '::ceilometer':
        debug                      => true,
        telemetry_secret           => $pass_ceilometer_telemetry,
        default_transport_url      => os_transport_url({
          'transport' => 'rabbit',
          'hosts'     => fqdn_rotate($all_masters),
          'port'      => '5671',
          'username'  => 'ceilometer',
          'password'  => $pass_ceilometer_messaging,
        }),
        notification_transport_url => os_transport_url({
          'transport' => 'rabbit',
          'hosts'     => fqdn_rotate($notif_rabbit_servers),
          'port'      => '5671',
          'username'  => 'ceilometer',
          'password'  => $pass_ceilometer_messaging,
        }),
        rabbit_use_ssl             => $use_ssl,
        kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
        amqp_sasl_mechanisms       => 'PLAIN',
      }
    }else{
      class { '::ceilometer':
        telemetry_secret           => $pass_ceilometer_telemetry,
        default_transport_url      => os_transport_url({
          'transport' => 'rabbit',
          'hosts'     => fqdn_rotate($all_masters),
          'port'      => '5671',
          'username'  => 'ceilometer',
          'password'  => $pass_ceilometer_messaging,
        }),
        notification_transport_url => os_transport_url({
          'transport' => 'rabbit',
          'hosts'     => fqdn_rotate($notif_rabbit_servers),
          'port'      => '5671',
          'username'  => 'ceilometer',
          'password'  => $pass_ceilometer_messaging,
        }),
        rabbit_use_ssl             => $use_ssl,
        kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
        amqp_sasl_mechanisms       => 'PLAIN',
      }
    }
    if($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri'){
      class { '::ceilometer::keystone::authtoken':
        password             => $pass_ceilometer_authtoken,
        auth_url             => $keystone_admin_uri,
        www_authenticate_uri => $keystone_auth_uri,
        cafile               => $api_endpoint_ca_file,
        region               => $region_name,
      }
    }
    if $openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria'{
      class { '::ceilometer::agent::auth':
        auth_password => $pass_ceilometer_authtoken,
        auth_url      => $keystone_auth_uri,
        auth_cacert   => $api_endpoint_ca_file,
        auth_region   => $region_name,
      }
    }else{
      class { '::ceilometer::agent::service_credentials':
        password    => $pass_ceilometer_authtoken,
        auth_url    => $keystone_auth_uri,
        cafile      => $api_endpoint_ca_file,
        region_name => $region_name,
      }
    }
    # Looks like the above sets ca_file instead of cafile.
    if $openstack_release == 'rocky' or $openstack_release == 'stein'{
      ceilometer_config {
        'service_credentials/cafile': value => $api_endpoint_ca_file;
      }
    }
    if $openstack_release == 'rocky' or $openstack_release == 'stein'{
      class { '::ceilometer::agent::compute':
        instance_discovery_method => 'libvirt_metadata',
      }
    }else{
      $compute_pollsters_list = [
                        # Libvirt / compute
                              'cpu',
                              'cpu.delta',
                              'vcpus',
                              'memory.usage',
                              'network.incoming.bytes',
                              'network.incoming.packets',
                              'network.incoming.bytes.rate',
                              'network.outgoing.bytes',
                              'network.outgoing.packets',
                              'network.outgoing.bytes.rate',
                              'disk.read.bytes',
                              'disk.read.bytes.rate',
                              'disk.read.requests',
                              'disk.read.requests.rate',
                              'disk.write.bytes',
                              'disk.write.bytes.rate',
                              'disk.write.requests',
                              'disk.write.requests.rate',
                              'disk.device.read.requests',
                              'disk.device.read.requests.rate',
                              'disk.device.write.requests',
                              'disk.device.write.requests.rate',
                              'disk.device.read.bytes',
                              'disk.device.read.bytes.rate',
                              'disk.device.write.bytes',
                              'disk.device.write.bytes.rate',
                              'ceilometer.instance.poller',
                              ]
      class { '::ceilometer::agent::polling':
        central_namespace         => false,
        compute_namespace         => true,
        ipmi_namespace            => false,
        instance_discovery_method => 'libvirt_metadata',
        manage_polling            => true,
        polling_interval          => 300,
        polling_meters            => $compute_pollsters_list,
      }
    }
    package { 'ceilometer-instance-poller':
      ensure => installed,
    }
  } else {
    package { 'ceilometer-instance-poller':
      ensure => purged,
    }
    package { 'python3-ceilometer':
      ensure => purged,
    }
  }
  ############################
  ### Install Manila-share ###
  ############################
  # Note: manila-share has to be setup in a compute node, because
  # it needs access to br-int and OVS.
  # It currently doesn't support HA, so it must be installed in a single
  # compute node only.
  if $has_subrole_manila and $manila_share_install and $install_manila{
    if $disable_notifications {
      $manila_notif_transport_url = ''
    }else{
      $manila_notif_transport_url = os_transport_url({
                                       'transport' => $messaging_notify_proto,
                                       'hosts'     => fqdn_rotate($notif_rabbit_servers),
                                       'port'      => $messaging_notify_port,
                                       'username'  => 'manila',
                                       'password'  => $pass_manila_messaging,
                                     })
    }
    class { '::manila::db':
      database_connection   => "mysql+pymysql://manila:${pass_manila_db}@${sql_vip_ip}/maniladb?charset=utf8",
    }
    class { '::manila':
      default_transport_url      => os_transport_url({
        'transport' => $messaging_default_proto,
        'hosts'     => fqdn_rotate($all_masters),
        'port'      => $messaging_default_port,
        'username'  => 'manila',
        'password'  => $pass_manila_messaging,
      }),
      notification_transport_url => $manila_notif_transport_url,
      notification_driver        => $notification_driver,
      rabbit_use_ssl             => $use_ssl,
      rabbit_ha_queues           => true,
      kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
      host                       => $machine_hostname,
    }
    # This *must* be called before ::manila::api that contains an include
    class { '::manila::keystone::authtoken':
      password             => $pass_manila_authtoken,
      user_domain_name     => 'Default',
      project_domain_name  => 'Default',
      auth_url             => $keystone_admin_uri,
      www_authenticate_uri => $keystone_auth_uri,
      memcached_servers    => $memcached_servers,
      cafile               => $api_endpoint_ca_file,
      region_name          => $region_name,
    }
    class { '::manila::compute::nova':
      cafile      => $api_endpoint_ca_file,
      username    => 'nova',
      password    => $pass_nova_authtoken,
      region_name => $region_name,
      auth_url    => $keystone_admin_uri,
      auth_type   => 'password',
    }
    class { '::manila::image::glance':
      cafile      => $api_endpoint_ca_file,
      region_name => $region_name,
      username    => 'glance',
      password    => $pass_glance_authtoken,
      auth_url    => $keystone_admin_uri,
      auth_type   => 'password',
    }
    class { '::manila::network::neutron':
      cafile      => $api_endpoint_ca_file,
      region_name => $region_name,
      username    => 'neutron',
      password    => $pass_neutron_authtoken,
      auth_url    => $keystone_admin_uri,
      auth_type   => 'password',
    }
    class { '::manila::volume::cinder':
      cafile      => $api_endpoint_ca_file,
      region_name => $region_name,
      username    => 'cinder',
      password    => $pass_cinder_authtoken,
      auth_url    => $keystone_admin_uri,
      auth_type   => 'password',
    }
    class { '::manila::share': }
    # manila-share cannot be setup on the controllers,
    # because it needs access to OVS.
    #class { '::manila::share': }
    class { '::manila::backends':
      enabled_share_backends => ['generic'],
    }
    # Deploy the generic driver using driver_handles_share_servers=True (ie: in VM shares...)
    ::manila::backend::generic { 'generic':
      driver_handles_share_servers => true,
      share_backend_name           => 'GENERIC',
    }
    ::manila::service_instance { 'generic':
      service_instance_user           => 'debian',
      service_instance_password       => 'do-not-use-passwords',
      service_image_name              => 'manila-service-image',
      create_service_image            => false,
      manila_service_keypair_name     => 'manila-ssh-key',
      path_to_public_key              => '/etc/manila/.ssh/id_rsa.pub',
      path_to_private_key             => '/etc/manila/.ssh/id_rsa',
      service_instance_flavor_id      => '100',
      service_instance_security_group => 'manila-security-group',
      service_network_name            => 'manila-service-network',
      service_network_cidr            => $manila_network_cidr,
      service_network_division_mask   => $manila_network_division_mask,
    }
    manila_config {
      "DEFAULT/my_ip": value => $machine_ip;
    }
  }

  ############################
  ### OpenVSwitch defaults ###
  ############################
  file { '/etc/default/openvswitch-switch':
    content => '# Managed by Puppet
OVS_CTL_OPTS="--delete-bridges"',
  }
}
