# a messaging node (for notifications)
#
class oci::messaging(
  $region_name               = 'RegionOne',
  $openstack_release         = undef,
  $cluster_name              = undef,
  $machine_hostname          = undef,
  $machine_ip                = undef,
  $machine_iface             = undef,

  $all_masters               = undef,
  $all_masters_ip            = undef,
  $first_master              = undef,
  $first_master_ip           = undef,

  $vip_hostname              = undef,
  $vip_ipaddr                = undef,
  $vip_netmask               = undef,

  $sql_vip_ip                = undef,
  $sql_vip_netmask           = undef,
  $sql_vip_iface             = undef,

  $is_first_rabbit           = false,
  $first_rabbit              = undef,
  $all_rabbits               = [],
  $all_rabbits_ips           = [],
  $all_rabbits_ids           = [],

  $initial_cluster_setup     = false,

  $messaging_vip_ipaddr      = undef,
  $messaging_vip_network     = undef,
  $messaging_vip_cidr        = undef,

  $messaging_other_nodes     = undef,
  $messaging_other_nodes_ips = undef,

  $has_subrole_heat          = true,
  $has_subrole_glance        = false,
  $has_subrole_nova          = false,
  $has_subrole_neutron       = false,
  $has_subrole_swift         = true,
  $has_subrole_horizon       = true,
  $has_subrole_barbican      = true,
  $has_subrole_cinder        = false,
  $has_subrole_gnocchi       = false,
  $has_subrole_ceilometer    = false,
  $has_subrole_panko         = false,
  $has_subrole_cloudkitty    = false,
  $has_subrole_aodh          = false,
  $has_subrole_octavia       = false,
  $has_subrole_magnum        = false,
  $has_subrole_manila        = false,
  $has_subrole_designate     = false,

  $use_ssl                   = true,
  $rabbit_env                = {},
  $pass_haproxy_stats        = undef,
  $pass_mysql_rootuser       = undef,
  $pass_mysql_backup         = undef,
  $pass_rabbitmq_cookie      = undef,
  $pass_rabbitmq_monitoring  = undef,
  $pass_keystone_messaging   = undef,
  $pass_nova_messaging       = undef,
  $pass_glance_messaging     = undef,
  $pass_cinder_messaging     = undef,
  $pass_neutron_messaging    = undef,
  $pass_heat_messaging       = undef,
  $pass_barbican_messaging   = undef,
  $pass_gnocchi_messaging    = undef,
  $pass_panko_messaging      = undef,
  $pass_ceilometer_messaging = undef,
  $pass_cloudkitty_messaging = undef,
  $pass_aodh_messaging       = undef,
  $pass_octavia_messaging    = undef,
  $pass_magnum_messaging     = undef,
  $pass_manila_messaging     = undef,
  $pass_designate_messaging  = undef,
  $pass_swift_messaging      = undef,
  $pass_zabbix_authtoken     = undef,

  $pass_gnocchi_db           = undef,
  $pass_gnocchi_authtoken    = undef,
  $pass_gnocchi_rscuuid      = undef,
  $pass_keystone_adminuser   = undef,

  $cloudkitty_metrics_yml    = undef,
  $cloudkitty_http_pool_maxsize = 10,

  $pass_cloudkitty_db        = undef,
  $pass_cloudkitty_authtoken = undef,

  $ceph_fsid                 = undef,
  $ceph_bootstrap_osd_key    = undef,
  $ceph_admin_key            = undef,
  $ceph_openstack_key        = undef,
  $ceph_mon_key              = undef,
  $ceph_mon_initial_members  = undef,
  $ceph_mon_host             = undef,

  # Zookeeper ensemble
  $zookeeper_ensemble_string = join($all_masters_ip,':2181,'),

  # Variables from variables.json
  $messaging_node_memcache_memory_limit    = 20,
  $messaging_node_memcache_max_connections = 65536,

  $kernel_from_backports    = false,

  $haproxy_timeout_http_request = '10s',
  $haproxy_timeout_queue        = '1m',
  $haproxy_timeout_connect      = '10s',
  $haproxy_timeout_client       = '1m',
  $haproxy_timeout_server       = '1m',
  $haproxy_timeout_check        = '10s',
){
  # Generic performances tweak (will default to throughput-performance)
  # class { '::oci::tuned': }

  $proto = 'https'
  $messaging_default_port = '5671'
  $messaging_notify_port = '5671'
  $api_port = 443
  $base_url = "${proto}://${vip_hostname}"

  $keystone_auth_uri  = "${base_url}:${api_port}/identity"
  $keystone_admin_uri = "${base_url}:${api_port}/identity"
  $messaging_all_nodes_ips = concat([$machine_ip], $messaging_other_nodes_ips)
  $memcached_string = join([join($messaging_all_nodes_ips,':11211,'), ':11211'],'')
  $memcached_servers  = ["${memcached_string}"]

  if $self_signed_api_cert {
    $api_endpoint_ca_file = "/etc/ssl/certs/oci-pki-oci-ca-chain.pem"
  }else{
    $api_endpoint_ca_file = ''
  }
  $oci_pki_root_ca_file = '/etc/ssl/certs/oci-pki-oci-ca-chain.pem'

  class { '::oci::puppet_oci_ca_cert':
    self_signed_api_cert => $self_signed_api_cert,
  }

  $sql_host = $messaging_vip_ipaddr

  if $facts['os']['lsb'] != undef{
    $mycodename = $facts['os']['lsb']['distcodename']
  }else{
    $mycodename = $facts['os']['distro']['codename']
  }

  ensure_resource('group', 'zabbix', {
    'ensure' => 'present',
    'gid'    => '966',
  })

  ensure_resource('file', '/etc/oci/monitoring_openstack_api_password', {
    'ensure'  => 'present',
    'content' => "${pass_zabbix_authtoken}",
    'group'   => 'zabbix',
    'mode'    => '0640',
  })

  ensure_resource('file', '/etc/oci/monitoring_openstack_api_url', {
    'ensure'  => 'present',
    'content' => "https://${vip_hostname}/identity/v3",
    'group'   => 'zabbix',
    'mode'    => '0640',
  })

  ################################
  ### Add a virtual IP address ###
  ################################
  # Setup corosync
  if ($openstack_release == 'rocky' or $openstack_release == 'stein') {
    class { 'corosync':
      authkey                  => '/var/lib/puppet/ssl/certs/ca.pem',
      bind_address             => $machine_ip,
      unicast_addresses        => $all_rabbits_ips,
      cluster_name             => 'mycluster',
      enable_secauth           => true,
      set_votequorum           => true,
      quorum_members           => $all_rabbits_ips,
      quorum_members_ids       => $all_rabbits_ids,
      log_stderr               => false,
      log_function_name        => true,
      syslog_priority          => 'debug',
      debug                    => true,
    }
    corosync::service { 'pacemaker':
      version => '0',
    }->
    cs_property { 'stonith-enabled':
      value   => 'false',
    }->
    cs_property { 'no-quorum-policy':
      value   => 'stop',
    }
  } else {
    class { 'corosync':
      authkey                  => '/var/lib/puppet/ssl/certs/ca.pem',
      bind_address             => $machine_ip,
      cluster_name             => 'mycluster',
      enable_secauth           => true,
      set_votequorum           => true,
      quorum_members           => $all_rabbits_ips,
      quorum_members_ids       => $all_rabbits_ids,
      log_stderr               => false,
      log_function_name        => true,
      syslog_priority          => 'debug',
      debug                    => true,
      enable_totem_interface   => false,
    }
    corosync::service { 'pacemaker':
      version => '0',
    }->
    cs_property { 'stonith-enabled':
      value   => 'false',
    }->
    cs_property { 'no-quorum-policy':
      value   => 'stop',
    }
  }
  if $messaging_vip_cidr == 32 {
    $vip_messaging_iface = 'lo'
  } else {
    $vip_messaging_iface = $sql_vip_iface
  }
  if $initial_cluster_setup {
    cs_primitive { 'openstack-messaging-vip':
      primitive_class => 'ocf',
      primitive_type  => 'IPaddr2',
      provided_by     => 'heartbeat',
      parameters      => { 'ip' => $messaging_vip_ipaddr, 'cidr_netmask' => $messaging_vip_cidr, 'nic' => $vip_messaging_iface },
      operations      => { 'monitor' => { 'interval' => '10s' } },
      require         => Cs_property['no-quorum-policy'],
    }
  }

  #####################
  ### Setup haproxy ###
  #####################
  # Here, we assume that stretch & buster don't have
  # a version of haproxy >= 2.1.
  if $mycodename == 'stretch' or $mycodename == 'buster' {
    # This used to work with haproxy 1.8,
    # let's see if we can get rid of it
    $haproxy_replace_path = 'reqrep'
    $haproxy_replace_head = 'rspirep'
    $use_haproxy_21_format = false
  }else{
    # This works with haproxy >= 2.1
    $haproxy_replace_path  = 'http-request replace-path'
    $haproxy_replace_head  = 'http-response replace-header'
    $use_haproxy_21_format = true
  }

  $haproxy_schedule = ['gnocchi::db::begin', 'cloudkitty::db::begin'];

  class { 'haproxy':
    global_options   => {
      'log'     => '/dev/log local0',
      'maxconn' => '40960',
      'user'    => 'haproxy',
      'group'   => 'haproxy',
      'stats'   => [
        'socket /var/lib/haproxy/stats',
        'timeout 30s'
      ],
      'daemon'   => '',
      'nbthread' => '4',
      'ssl-default-bind-ciphers'   => 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256',
      'ssl-default-bind-options'   => 'no-sslv3 no-tlsv10 no-tlsv11 no-tls-tickets',
      'ssl-default-server-ciphers' => 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256',
      'ssl-default-server-options' => 'no-sslv3 no-tlsv10 no-tlsv11 no-tls-tickets',
    },
    defaults_options => {
      'mode'      => 'http',
      'option'    => [
          'httplog',
        ],
      'timeout' => [
          "http-request ${haproxy_timeout_http_request}",
          "queue ${haproxy_timeout_queue}",
          "connect ${haproxy_timeout_connect}",
          "client ${haproxy_timeout_client}",
          "server ${haproxy_timeout_server}",
          "check ${haproxy_timeout_check}",
        ],
      'stats'     => 'enable',
    },
    merge_options => true,
    before        => Anchor[$haproxy_schedule],
    require       => Sysctl::Value['net.ipv4.ip_nonlocal_bind'],
  }->
  haproxy::listen { 'hapstats':
    section_name => 'statshttp',
    bind         => { "${machine_ip}:8088" => 'user root'},
    mode         => 'http',
    options      => {
        'stats' => [ 'uri /', "auth admin:${pass_haproxy_stats}", 'refresh 15', 'admin if TRUE', 'realm Haproxy Statistics', ],
      },
  }->
  haproxy::listen { 'haphealthcheck':
    section_name => 'healthcheck',
    bind         => { "${machine_ip}:8081" => 'user root'},
    mode         => 'health',
    options => [
       { 'option' => 'httpchk' },
    ]
  }

  # Define galera frontend and backend
  haproxy::frontend { 'galerafe':
    mode      => 'tcp',
    bind      => { "${messaging_vip_ipaddr}:3306" => [] },
    options   => [
      { 'timeout'         => 'client 3600s'},
      { 'default_backend' => 'galerabe'},
    ],
    before => Anchor[$haproxy_schedule],
  }
  haproxy::backend { 'galerabe':
    options => [
      { 'mode'    => 'tcp' },
      { 'balance' => 'roundrobin' },
      { 'timeout' => 'check 5000' },
      { 'timeout' => 'server 3600s' },
      { 'option'  => 'httpchk'},
    ],
    before => Anchor[$haproxy_schedule],
  }
  haproxy::balancermember { 'galerabm':
    listening_service => 'galerabe',
    ipaddresses       => $machine_ip,
    server_names      => $machine_hostname,
    options           => 'check inter 4000 port 9200 fall 3 rise 5',
    before => Anchor[$haproxy_schedule],
  }
  haproxy::balancermember { 'galerabmback':
    listening_service => 'galerabe',
    ipaddresses       => $messaging_other_nodes_ips,
    server_names      => $messaging_other_nodes,
    options           => 'check inter 4000 port 9200 fall 3 rise 5 backup',
    before => Anchor[$haproxy_schedule],
  }

  #######################
  ### Setup memcached ###
  #######################
  class { '::memcached':
    listen_ip       => $machine_ip,
    udp_port        => 0,
    max_memory      => "${messaging_node_memcache_memory_limit}%",
    max_connections => $messaging_node_memcache_max_connections,
    pidfile         => '/var/run/memcached/memcached.pid',
    user            => 'memcache',
  }

  #######################
  ### Setup databases ###
  #######################
  if $initial_cluster_setup {
    if $mycodename != 'stretch'{
      package { 'mariadb-backup':
        ensure => present,
        before => Class['galera'],
      }
      $wsrep_sst_method = 'mariabackup'
    }else{
      $wsrep_sst_method = 'mariabackup'
    }

    if $is_first_rabbit {
      # If this is the first node, it must publish itself as working
      # before waiting for the other nodes to join.
      Class['galera::status'] -> Exec['galera-size-is-correct']
      Service['xinetd'] -> Exec['galera-size-is-correct']
    }else{
      exec {'galera-first-master-is-up':
      command => "/usr/bin/oci-wait-for-first-galera-node ${$first_rabbit} 4800",
        unless  => '/bin/false # comment to satisfy puppet syntax requirements',
        timeout => 5000,
      }
      Exec['galera-first-master-is-up'] -> Class['galera']
    }

    if $mycodename == 'stretch' or $mycodename == 'buster' {
      $galera_package_name = 'galera-3'
    }else{
      $galera_package_name = 'galera-4'
    }

    $ram_total_in_mb = Integer($facts['memorysize_mb'])
    $ram_total_in_gb = $ram_total_in_mb / 1024
    $ram_total_in_gb_div_4 = $ram_total_in_gb / 4

    if $ram_total_in_gb_div_4 > 64 {
      $innodb_buffer_pool_size = 64
    }elsif $ram_total_in_gb_div_4 > 32{
      $innodb_buffer_pool_size = 32
    }elsif $ram_total_in_gb_div_4 > 16{
      $innodb_buffer_pool_size = 16
    }elsif $ram_total_in_gb_div_4 > 8{
      $innodb_buffer_pool_size = 8
    }elsif $ram_total_in_gb_div_4 > 4{
      $innodb_buffer_pool_size = 4
    }else{
      $innodb_buffer_pool_size = 2
    }

    class { 'galera':
      galera_servers      => $all_rabbits_ips,
      galera_master       => $first_rabbit,
      local_ip            => $machine_ip,
      mysql_package_name  => 'mariadb-server',
      client_package_name => 'default-mysql-client',
      vendor_type         => 'mariadb',
      root_password       => $pass_mysql_rootuser,
      status_password     => $pass_mysql_rootuser,
      deb_sysmaint_password => $pass_mysql_rootuser,
      configure_repo      => false,
      configure_firewall  => false,
      galera_package_name => $galera_package_name,
      override_options => {
        'mysqld' => {
          'bind_address'                    => $machine_ip,
          'wait_timeout'                    => '28800',
          'interactive_timeout'             => '30',
          'connect_timeout'                 => '30',
          'character_set_server'            => 'utf8',
          'collation_server'                => 'utf8_general_ci',
          'innodb_buffer_pool_size'         => "${innodb_buffer_pool_size}G",
          'innodb_flush_log_at_trx_commit'  => '2',
          'max_connections'                 => '2000',
          'max_user_connections'            => '1000',
          'binlog_cache_size'               => '1M',
          'log-bin'                         => 'mysql-bin',
          'binlog_format'                   => 'ROW',
          'performance_schema'              => '1',
          'log_warnings'                    => '2',
          'wsrep_sst_auth'                  => "backup:${pass_mysql_backup}",
          'wsrep_sst_method'                => $wsrep_sst_method,
          'wsrep_cluster_name'              => $cluster_name,
          'wsrep_node_name'                 => $machine_hostname,
        }
      },
      require             => Cs_primitive['openstack-messaging-vip'],
    }->
    mysql_user { 'backup@%':
      ensure        => present,
      password_hash => mysql_password($pass_mysql_backup),
    }->
    mysql_grant{'backup@%/*.*':
      ensure     => 'present',
      options    => ['GRANT'],
      privileges => ['SELECT', 'PROCESS', 'CREATE TEMPORARY TABLES', 'LOCK TABLES', 'SHOW VIEW', 'RELOAD', 'TRIGGER', 'REPLICATION CLIENT'],
      table      => '*.*',
      user       => 'backup@%',
    }->

    # Wait until SHOW STATUS LIKE 'wsrep_cluster_status' shows Primary
    exec {'galera-is-up':
      command => "/usr/bin/oci-wait-for-sql \"SHOW STATUS LIKE 'wsrep_cluster_status'\" Primary mysql 4800",
      unless  => '/bin/false # comment to satisfy puppet syntax requirements',
      timeout => 5000,
    }

    # Wait until SHOW STATUS LIKE 'wsrep_connected' shows ON
    exec {'galera-wsrep-connected-on':
      command => "/usr/bin/oci-wait-for-sql \"SHOW STATUS LIKE 'wsrep_connected'\" ON mysql 4800",
      unless  => '/bin/false # comment to satisfy puppet syntax requirements',
      require => Exec['galera-is-up'],
      timeout => 5000,
    }

    # Wait until SHOW STATUS LIKE 'wsrep_local_state_comment' shows Synced
    exec {'galera-is-synced':
      command => "/usr/bin/oci-wait-for-sql \"SHOW STATUS LIKE 'wsrep_local_state_comment'\" Synced mysql 4800",
      unless  => '/bin/false # comment to satisfy puppet syntax requirements',
      require => Exec['galera-wsrep-connected-on'],
      timeout => 5000,
    }

    # Wait until all nodes are connected to the cluster
    $galera_cluster_num_of_nodes = sprintf('%i', $all_masters.size)
    exec {'galera-size-is-correct':
      command => "/usr/bin/oci-wait-for-sql \"SHOW STATUS LIKE 'wsrep_cluster_size'\" ${galera_cluster_num_of_nodes} mysql 4800",
      unless  => '/bin/false # comment to satisfy puppet syntax requirements',
      require => Exec['galera-is-synced'],
      timeout => 5000,
    }

    ######################################
    ### Create or wait for db creation ###
    ######################################
    # Gnocchi DB
    if $is_first_rabbit {
      class { '::gnocchi::db::mysql':
        dbname   => 'gnocchidb',
        password => $pass_gnocchi_db,
        allowed_hosts => '%',
        require  => Exec['galera-size-is-correct'],
        before   => Anchor['gnocchi::service::begin'],
      }
    } else {
      exec { 'gnocchi-db-user':
        command => "/usr/bin/oci-wait-for-sql \"SELECT User FROM user WHERE User='gnocchi'\" gnocchi mysql 4800",
        unless  => '/bin/false # comment to satisfy puppet syntax requirements',
        require => Exec['galera-size-is-correct'],
        before  => Anchor['gnocchi::service::begin'],
        timeout => 5000,
      }
    }

    # Cloudkitty DB
    if $is_first_rabbit {
      class { '::cloudkitty::db::mysql':
        dbname   => 'cloudkittydb',
        password => $pass_cloudkitty_db,
        allowed_hosts => '%',
        require  => Exec['galera-size-is-correct'],
        before   => Anchor['cloudkitty::service::begin'],
      }
    } else {
      exec { 'cloudkitty-db-user':
        command => "/usr/bin/oci-wait-for-sql \"SELECT User FROM user WHERE User='cloudkitty'\" cloudkitty mysql 4800",
        unless  => '/bin/false # comment to satisfy puppet syntax requirements',
        require => Exec['galera-size-is-correct'],
        before  => Anchor['cloudkitty::service::begin'],
        timeout => 5000,
      }
    }
  }

  ##################
  ### Setup Ceph ###
  ##################
  class { 'ceph':
    fsid                => $ceph_fsid,
    ensure              => 'present',
    authentication_type => 'cephx',
    mon_initial_members => $ceph_mon_initial_members,
    mon_host            => $ceph_mon_host,
  }->
  ceph::key { 'client.admin':
    secret  => $ceph_admin_key,
    cap_mon => 'allow *',
    cap_osd => 'allow *',
    cap_mds => 'allow',
  }->
  ceph::key { 'client.openstack':
    secret  => $ceph_openstack_key,
    mode    => '0644',
    cap_mon => 'profile rbd',
    cap_osd => 'profile rbd pool=gnocchi',
  }
  $ceph_pools = ['gnocchi']
  ceph::pool { $ceph_pools: }
  class { '::ceph::profile::params':
    fsid                => $ceph_fsid,
    manage_repo         => false,
    release             => 'nautilus',
    authentication_type => 'cephx',
    mon_host            => $ceph_mon_host,
    mon_initial_members => $ceph_mon_initial_members,
    client_keys         => {
      'client.admin' => {
        secret  => $ceph_admin_key,
        cap_mon => 'allow *',
        cap_osd => 'allow *',
        cap_mds => 'allow',
      },
      'client.openstack' => {
        secret  => $ceph_openstack_key,
        mode    => '0644',
        cap_mon => 'profile rbd',
        cap_osd => 'profile rbd pool=gnocchi',
      }
    }
  }

  #####################
  ### Setup Gnocchi ###
  #####################
  oci::sslkeypair {'gnocchi':
    notify_service_name => 'gnocchi-api',
  }
  $gnocchi_key_file = "/etc/gnocchi/ssl/private/${::fqdn}.pem"
  $gnocchi_crt_file = "/etc/gnocchi/ssl/public/${::fqdn}.crt"
  if $openstack_release == 'rocky'{
    class { '::gnocchi':
      debug                 => true,
      database_connection   => "mysql+pymysql://gnocchi:${pass_gnocchi_db}@${sql_host}/gnocchidb?charset=utf8",
    }
  }else{
    if ($openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri') {
      class { '::gnocchi':
        database_connection   => "mysql+pymysql://gnocchi:${pass_gnocchi_db}@${sql_host}/gnocchidb?charset=utf8",
      }
    } else {
      class { '::gnocchi':
        coordination_url => "zookeeper://${zookeeper_ensemble_string}:2181/",
      }
      class { 'gnocchi::db':
        database_connection => "mysql+pymysql://gnocchi:${pass_gnocchi_db}@${sql_host}/gnocchidb?charset=utf8",
      }
    }
    class { '::gnocchi::logging':
      debug => true,
    }
  }

  class { '::gnocchi::keystone::authtoken':
    password             => $pass_gnocchi_authtoken,
    auth_url             => $keystone_admin_uri,
    www_authenticate_uri => $keystone_auth_uri,
    memcached_servers    => $memcached_servers,
    token_cache_time     => 3600,
    cafile               => $api_endpoint_ca_file,
  }
  if $is_first_rabbit {
    class { '::gnocchi::api':
      enabled                      => true,
      service_name                 => 'gnocchi-api',
      sync_db                      => true,
      enable_proxy_headers_parsing => true,
    }
  }else{
    class { '::gnocchi::api':
      enabled                      => true,
      service_name                 => 'gnocchi-api',
      sync_db                      => false,
      enable_proxy_headers_parsing => true,
    }
  }
  class { '::gnocchi::client': }
  # Fix the number of uwsgi processes
  $gnocchi_workers = $::os_workers * 2
  class { '::gnocchi::wsgi::uwsgi':
    processes => $gnocchi_workers,
  }
  if ($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri') {
    class { '::gnocchi::metricd':
      workers       => $::os_workers,
      cleanup_delay => 20,
    }
  }elsif $openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena'{
    class { '::gnocchi::metricd':
      workers                 => $::os_workers,
      cleanup_delay           => 20,
      metric_processing_delay => 10,
    }
  }else{
    class { '::gnocchi::metricd':
      workers                 => $::os_workers,
      metric_cleanup_delay    => 20,
      metric_processing_delay => 10,
    }
  }
  if ($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri') {
    class { '::gnocchi::storage':
      metric_processing_delay => 10,
      coordination_url        => "zookeeper://${zookeeper_ensemble_string}:2181/",
    }
  }
  if $openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena'{
    class { '::gnocchi::storage::ceph':
      ceph_username => 'openstack',
      ceph_keyring  => '/etc/ceph/ceph.client.openstack.keyring',
      manage_cradox => false,
      manage_rados  => true,
    }
  }else{
    class { '::gnocchi::storage::ceph':
      ceph_username => 'openstack',
      ceph_keyring  => '/etc/ceph/ceph.client.openstack.keyring',
      manage_rados  => true,
    }
  }

  gnocchi_config {
    'database/connection': value => "mysql+pymysql://gnocchi:${pass_gnocchi_db}@${sql_host}/gnocchidb?charset=utf8", secret => true;
  }

  class { '::gnocchi::statsd':
    archive_policy_name => 'ik-medium-rate',
    flush_delay         => '100',
    # random datas:
    resource_id         => $pass_gnocchi_rscuuid,
  }

  ########################
  ### Setup CloudKitty ###
  ########################
  if $initial_cluster_setup {
    Class['galera'] -> Anchor['cloudkitty::install::begin']
  }
  oci::sslkeypair {'cloudkitty':
    notify_service_name => 'cloudkitty-api',
  }
  $cloudkitty_key_file = "/etc/cloudkitty/ssl/private/${::fqdn}.pem"
  $cloudkitty_crt_file = "/etc/cloudkitty/ssl/public/${::fqdn}.crt"
  if $openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train'{
    class { '::cloudkitty::db':
      database_connection   => "mysql+pymysql://cloudkitty:${pass_cloudkitty_db}@${sql_host}/cloudkittydb?charset=utf8",
      database_idle_timeout => 1800,
    }
  }else{
    class { '::cloudkitty::db':
      database_connection              => "mysql+pymysql://cloudkitty:${pass_cloudkitty_db}@${sql_host}/cloudkittydb?charset=utf8",
      database_connection_recycle_time => 1800,
    }
  }
  $cloudkitty_notif_transport_url = os_transport_url({
                                      'transport' => 'rabbit',
                                      'hosts'     => fqdn_rotate($all_rabbits),
                                      'port'      => '5671',
                                      'username'  => 'cloudkitty',
                                      'password'  => $pass_cloudkitty_messaging,
                                    })

  if $openstack_release == 'rocky'{
    class { '::cloudkitty':
      default_transport_url      => os_transport_url({
                                                      'transport' => 'rabbit',
                                                      'hosts'     => fqdn_rotate($all_masters),
                                                      'port'      => '5671',
                                                      'username'  => 'cloudkitty',
                                                      'password'  => $pass_cloudkitty_messaging,
                                                    }),
      notification_transport_url => $cloudkitty_notif_transport_url,
      rabbit_use_ssl             => $use_ssl,
      rabbit_ha_queues           => true,
      kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
      amqp_sasl_mechanisms       => 'PLAIN',
      storage_backend            => 'sqlalchemy',
      tenant_fetcher_backend     => 'keystone',
      auth_section               => 'keystone_authtoken',
      host                       => $machine_hostname,
    }
  }else{
    if ($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri') {
      class { '::cloudkitty':
        default_transport_url      => os_transport_url({
                                                        'transport' => 'rabbit',
                                                        'hosts'     => fqdn_rotate($all_masters),
                                                        'port'      => '5671',
                                                        'username'  => 'cloudkitty',
                                                        'password'  => $pass_cloudkitty_messaging,
                                                      }),
        notification_transport_url => $cloudkitty_notif_transport_url,
        rabbit_use_ssl             => true,
        rabbit_ha_queues           => true,
        kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
        amqp_sasl_mechanisms       => 'PLAIN',
        storage_backend            => 'sqlalchemy',
        storage_version            => '1',
        tenant_fetcher_backend     => 'keystone',
        auth_section               => 'keystone_authtoken',
        host                       => $machine_hostname,
      }
    }else{
      if $openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena'{
        class { '::cloudkitty':
          default_transport_url      => os_transport_url({
                                                          'transport' => 'rabbit',
                                                          'hosts'     => fqdn_rotate($all_masters),
                                                          'port'      => '5671',
                                                          'username'  => 'cloudkitty',
                                                          'password'  => $pass_cloudkitty_messaging,
                                                        }),
          notification_transport_url => $cloudkitty_notif_transport_url,
          rabbit_use_ssl             => $use_ssl,
          rabbit_ha_queues           => true,
          kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
          amqp_sasl_mechanisms       => 'PLAIN',
          storage_backend            => 'sqlalchemy',
          storage_version            => '1',
          fetcher_backend            => 'keystone',
          auth_section               => 'ociadmin_authtoken',
          host                       => $machine_hostname,
        }
      }else{
        class { '::cloudkitty::fetcher::keystone':
          auth_section            => 'ociadmin_authtoken',
          ignore_rating_role      => true,
          ignore_disabled_tenants => true,
        }
        class { '::cloudkitty':
          default_transport_url      => os_transport_url({
                                                          'transport' => 'rabbit',
                                                          'hosts'     => fqdn_rotate($all_masters),
                                                          'port'      => '5671',
                                                          'username'  => 'cloudkitty',
                                                          'password'  => $pass_cloudkitty_messaging,
                                                        }),
          notification_transport_url => $cloudkitty_notif_transport_url,
          rabbit_use_ssl             => $use_ssl,
          rabbit_ha_queues           => true,
          kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
          amqp_sasl_mechanisms       => 'PLAIN',
          storage_backend            => 'sqlalchemy',
          storage_version            => '1',
          fetcher_backend            => 'keystone',
          auth_section               => 'ociadmin_authtoken',
          host                       => $machine_hostname,
        }
      }
      class { '::cloudkitty::logging':
        debug => true,
      }
    }
  }

  class { '::cloudkitty::keystone::authtoken':
    password             => $pass_cloudkitty_authtoken,
    auth_url             => $keystone_admin_uri,
    www_authenticate_uri => $keystone_auth_uri,
    memcached_servers    => $memcached_servers,
    token_cache_time     => 3600,
    cafile               => $api_endpoint_ca_file,
    region_name          => $region_name,
  }
  class { '::cloudkitty::client': }
  class { '::cloudkitty::api':
    sync_db      => $do_db_sync,
  }
  # Fix the number of uwsgi processes
  class { '::cloudkitty::wsgi::uwsgi': }
  if $openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena'{
    cloudkitty_config {
      'fetcher_keystone/ignore_disabled_tenants':  value => 'True';
      'fetcher_keystone/ignore_rating_role':       value => 'True';
      'orchestrator/coordination_url':             value => "zookeeper://${zookeeper_ensemble_string}:2181/";
    }
  }
  cloudkitty_config {
    'keystone_fetcher/cafile':             value => $api_endpoint_ca_file;
    'gnocchi_collector/cafile':            value => $api_endpoint_ca_file;
    'coordination/backend_url':            value => "zookeeper://${zookeeper_ensemble_string}:2181/";
    'fetcher_gnocchi/cafile':              value => $api_endpoint_ca_file;
    'fetcher_gnocchi/auth_section':        value => 'keystone_authtoken';
    'fetcher_gnocchi/http_pool_maxsize':   value => $cloudkitty_http_pool_maxsize;
    'collector_gnocchi/http_pool_maxsize': value => $cloudkitty_http_pool_maxsize;
  }

  # This is needed for the Keystone fetcher to list users
  cloudkitty_config {
    'ociadmin_authtoken/auth_type':            value => 'password';
    'ociadmin_authtoken/auth_url':             value => $keystone_admin_uri;
    'ociadmin_authtoken/project_name':         value => 'admin';
    'ociadmin_authtoken/project_domain_name':  value => 'Default';
    'ociadmin_authtoken/username':             value => 'admin';
    'ociadmin_authtoken/user_domain_name':     value => 'Default';
    'ociadmin_authtoken/password':             value => $pass_keystone_adminuser;
    'ociadmin_authtoken/www_authenticate_uri': value => $keystone_auth_uri;
    'ociadmin_authtoken/cafile':               value => $api_endpoint_ca_file;
    'ociadmin_authtoken/memcached_servers':    value => $memcached_string;
    'ociadmin_authtoken/region_name':          value => $region_name;
  }
  if ($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri') {
    cloudkitty_config {
      'fetcher/backend': value => 'keystone';
    }
  }

  if $cloudkitty_metrics_yml {
    file { "/etc/cloudkitty/metrics.yml":
      ensure                  => present,
      owner                   => root,
      group                   => root,
      content                 => base64('decode', $cloudkitty_metrics_yml),
      selinux_ignore_defaults => true,
      mode                    => '0644',
      require                 => Anchor['cloudkitty::install::end'],
      notify                  => Service['cloudkitty-processor'],
    }
  }

  if ($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby') {
    class { '::cloudkitty::processor':
      auth_section => 'ociadmin_authtoken',
    }
    cloudkitty_config {
      'collector_gnocchi/region_name': value => $region_name;
    }
  }else{
    if $openstack_release == 'wallaby' {
      class { '::cloudkitty::processor':
        auth_section => 'ociadmin_authtoken',
        region_name  => $region_name,
      }
    }else{
      class { '::cloudkitty::orchestrator':
        coordination_url => "zookeeper://${zookeeper_ensemble_string}:2181/",
      }
      class { '::cloudkitty::processor':
        auth_section => 'ociadmin_authtoken',
        region_name  => $region_name,
      }
    }
  }

  ######################
  ### Setup RabbitMQ ###
  ######################
  if $use_ssl {
    file { "/etc/rabbitmq/ssl/private":
      ensure                  => directory,
      owner                   => 'root',
      mode                    => '0755',
      require                 => File['/etc/rabbitmq/ssl'],
      selinux_ignore_defaults => true,
    }->
    file { "/etc/rabbitmq/ssl/public":
      ensure                  => directory,
      owner                   => 'root',
      mode                    => '0755',
      selinux_ignore_defaults => true,
    }->
    file { "/etc/rabbitmq/ssl/private/${::fqdn}.key":
      ensure                  => present,
      owner                   => "rabbitmq",
      source                  => "/etc/ssl/private/ssl-cert-snakeoil.key",
      selinux_ignore_defaults => true,
      mode                    => '0600',
    }->
    file { "/etc/rabbitmq/ssl/public/${::fqdn}.crt":
      ensure                  => present,
      owner                   => "rabbitmq",
      source                  => '/etc/ssl/certs/ssl-cert-snakeoil.pem',
      selinux_ignore_defaults => true,
      mode                    => '0644',
      notify        => Service['rabbitmq-server'],
    }
    $rabbit_ssl_cert = "/etc/rabbitmq/ssl/public/${::fqdn}.crt"
    $rabbit_ssl_key  = "/etc/rabbitmq/ssl/private/${::fqdn}.key"
    $rabbit_ssl_ca   = $oci_pki_root_ca_file
  } else {
    $rabbit_ssl_cert = UNSET
    $rabbit_ssl_key  = UNSET
    $rabbit_ssl_ca   = UNSET
  }

  class { '::rabbitmq':
    delete_guest_user           => true,
    node_ip_address             => $machine_ip,
    ssl_interface               => $machine_ip,
    ssl                         => $use_ssl,
    ssl_only                    => false,
    ssl_cacert                  => $rabbit_ssl_ca,
    ssl_cert                    => $rabbit_ssl_cert,
    ssl_key                     => $rabbit_ssl_key,
    environment_variables       => $rabbit_env,
    repos_ensure                => false,
    # Clustering options...
    config_cluster              => true,
    cluster_nodes               => $all_rabbits,
    cluster_node_type           => 'ram',
    erlang_cookie               => $pass_rabbitmq_cookie,
    wipe_db_on_cookie_change    => true,
    collect_statistics_interval => 60000,
    cluster_partition_handling  => 'autoheal',
    config_variables            => {
        'vm_memory_high_watermark'   => '0.4',
      },
  }->
  rabbitmq_vhost { '/':
    provider => 'rabbitmqctl',
    require  => Class['::rabbitmq'],
  }

  if $initial_cluster_setup {
    exec { 'auto-join-rabbit-cluster':
      command => "/usr/bin/oci-auto-join-rabbitmq-cluster ${first_rabbit}",
      unless  => "/bin/false",
      require => Class['::rabbitmq'],
    }
    if $is_first_rabbit {
      rabbitmq_user { 'keystone':
        admin    => true,
        password => $pass_keystone_messaging,
        require  => Class['::rabbitmq'],
      }->
      rabbitmq_user_permissions { "keystone@/":
        configure_permission => '.*',
        write_permission     => '.*',
        read_permission      => '.*',
        provider             => 'rabbitmqctl',
        require              => Class['::rabbitmq'],
      }

      rabbitmq_user { 'monitoring':
        admin    => true,
        password => $pass_rabbitmq_monitoring,
        require  => Class['::rabbitmq'],
      }->
      rabbitmq_user_permissions { "monitoring@/":
        configure_permission => '.*',
        write_permission     => '.*',
        read_permission      => '.*',
        provider             => 'rabbitmqctl',
      }


      if $has_subrole_glance {
        rabbitmq_user { 'glance':
          admin    => true,
          password => $pass_glance_messaging,
        }->
        rabbitmq_user_permissions { "glance@/":
          configure_permission => '.*',
          write_permission     => '.*',
          read_permission      => '.*',
          provider             => 'rabbitmqctl',
          require              => Class['::rabbitmq'],
        }
      }

      if $has_subrole_heat {
        rabbitmq_user { 'heat':
          admin    => true,
          password => $pass_heat_messaging,
        }->
        rabbitmq_user_permissions { "heat@/":
          configure_permission => '.*',
          write_permission     => '.*',
          read_permission      => '.*',
          provider             => 'rabbitmqctl',
          require              => Class['::rabbitmq'],
        }
      }

      if $has_subrole_barbican {
        rabbitmq_user { 'barbican':
          admin    => true,
          password => $pass_barbican_messaging,
        }->
        rabbitmq_user_permissions { "barbican@/":
          configure_permission => '.*',
          write_permission     => '.*',
          read_permission      => '.*',
          provider             => 'rabbitmqctl',
          require              => Class['::rabbitmq'],
        }
      }

      if $has_subrole_nova {
        rabbitmq_user { 'nova':
          admin    => true,
          password => $pass_nova_messaging,
        }->
        rabbitmq_user_permissions { "nova@/":
          configure_permission => '.*',
          write_permission     => '.*',
          read_permission      => '.*',
          provider             => 'rabbitmqctl',
          require              => Class['::rabbitmq'],
        }
      }
      if $has_subrole_neutron {
        rabbitmq_user { 'neutron':
          admin    => true,
          password => $pass_neutron_messaging,
        }->
        rabbitmq_user_permissions { "neutron@/":
          configure_permission => '.*',
          write_permission     => '.*',
          read_permission      => '.*',
          provider             => 'rabbitmqctl',
          require              => Class['::rabbitmq'],
        }
      }
      if $has_subrole_cinder {
        rabbitmq_user { 'cinder':
          admin    => true,
          password => $pass_cinder_messaging,
        }->
        rabbitmq_user_permissions { "cinder@/":
          configure_permission => '.*',
          write_permission     => '.*',
          read_permission      => '.*',
          provider             => 'rabbitmqctl',
          require              => Class['::rabbitmq'],
        }
      }
      if $has_subrole_gnocchi {
        rabbitmq_user { 'gnocchi':
          admin    => true,
          password => $pass_gnocchi_messaging,
        }->
        rabbitmq_user_permissions { "gnocchi@/":
          configure_permission => '.*',
          write_permission     => '.*',
          read_permission      => '.*',
          provider             => 'rabbitmqctl',
          require              => Class['::rabbitmq'],
        }
      }
      if $has_subrole_panko {
        rabbitmq_user { 'panko':
          admin    => true,
          password => $pass_panko_messaging,
        }->
        rabbitmq_user_permissions { "panko@/":
          configure_permission => '.*',
          write_permission     => '.*',
          read_permission      => '.*',
          provider             => 'rabbitmqctl',
          require              => Class['::rabbitmq'],
        }
      }
      if $has_subrole_ceilometer {
        rabbitmq_user { 'ceilometer':
          admin    => true,
          password => $pass_ceilometer_messaging,
        }->
        rabbitmq_user_permissions { "ceilometer@/":
          configure_permission => '.*',
          write_permission     => '.*',
          read_permission      => '.*',
          provider             => 'rabbitmqctl',
          require              => Class['::rabbitmq'],
        }
      }
      if $has_subrole_swift {
        rabbitmq_user { 'swift':
          admin    => true,
          password => $pass_swift_messaging,
        }->
        rabbitmq_user_permissions { "swift@/":
          configure_permission => '.*',
          write_permission     => '.*',
          read_permission      => '.*',
          provider             => 'rabbitmqctl',
          require              => Class['::rabbitmq'],
        }
      }
      if $has_subrole_cloudkitty {
        rabbitmq_user { 'cloudkitty':
          admin    => true,
          password => $pass_cloudkitty_messaging,
        }->
        rabbitmq_user_permissions { "cloudkitty@/":
          configure_permission => '.*',
          write_permission     => '.*',
          read_permission      => '.*',
          provider             => 'rabbitmqctl',
          require              => Class['::rabbitmq'],
        }
      }
      if $has_subrole_aodh {
        rabbitmq_user { 'aodh':
          admin    => true,
          password => $pass_aodh_messaging,
        }->
        rabbitmq_user_permissions { "aodh@/":
          configure_permission => '.*',
          write_permission     => '.*',
          read_permission      => '.*',
          provider             => 'rabbitmqctl',
          require              => Class['::rabbitmq'],
        }
      }
      if $has_subrole_octavia {
        rabbitmq_user { 'octavia':
          admin    => true,
          password => $pass_octavia_messaging,
        }->
        rabbitmq_user_permissions { "octavia@/":
          configure_permission => '.*',
          write_permission     => '.*',
          read_permission      => '.*',
          provider             => 'rabbitmqctl',
          require              => Class['::rabbitmq'],
        }
      }
      if $has_subrole_magnum {
        rabbitmq_user { 'magnum':
          admin    => true,
          password => $pass_magnum_messaging,
        }->
        rabbitmq_user_permissions { "magnum@/":
          configure_permission => '.*',
          write_permission     => '.*',
          read_permission      => '.*',
          provider             => 'rabbitmqctl',
          require              => Class['::rabbitmq'],
        }
      }
      if $has_subrole_manila {
        rabbitmq_user { 'manila':
          admin    => true,
          password => $pass_manila_messaging,
        }->
        rabbitmq_user_permissions { "manila@/":
          configure_permission => '.*',
          write_permission     => '.*',
          read_permission      => '.*',
          provider             => 'rabbitmqctl',
          require              => Class['::rabbitmq'],
        }
      }
      if $has_subrole_designate {
        rabbitmq_user { 'designate':
          admin    => true,
          password => $pass_designate_messaging,
        }->
        rabbitmq_user_permissions { "designate@/":
          configure_permission => '.*',
          write_permission     => '.*',
          read_permission      => '.*',
          provider             => 'rabbitmqctl',
          require              => Class['::rabbitmq'],
        }
      }
    }
  }
}
