Improve Pulsar AWS deployment (terraform + ansible)

driver-pulsar/README.md

@@ -3,13 +3,17 @@
 For instructions on running the OpenMessaging benchmarks for Pulsar, see the [official documentation](http://openmessaging.cloud/docs/benchmarks/pulsar/).
 
 ## Supplement to the official documentation
-
-Before you run `ansible-playbook` with `terraform-inventory`, you must set the environment variable `TF_STATE`. i.e. the completed command should be:
+For Ansible to have access to the inventory defined in the Terraform configuration, the Terraform Collection for Ansible is required.
+Install it with the following command ([source](https://mdawar.dev/blog/ansible-terraform-inventory)):
+```bash
+ansible-galaxy collection install cloud.terraform
+ansible-galaxy role install geerlingguy.docker
+```
 
 ```bash
-TF_STATE=. ansible-playbook \
+ansible-playbook \
   --user ec2-user \
-  --inventory `which terraform-inventory` \
+  --inventory terraform.yaml \
   deploy.yaml
 ```
 
@@ -20,19 +24,11 @@ The Ansible deployment script supports flexible configuration with a variable fi
 ```bash
 TF_STATE=. ansible-playbook \
   --user ec2-user \
-  --inventory `which terraform-inventory` \
+  --inventory terraform.yaml \
   -e @extra_vars.yaml \
   deploy.yaml
 ```
 
-For example, if you changed the AWS instance type, the two SSD device paths might not be `/dev/nvme1n1` and `/dev/nvme2n1`. In this case, you can configure them like
-
-```yaml
-disk_dev:
-  - /path/to/disk1
-  - /path/to/disk2
-```
-
 See more explanations in [the example variable file](./deploy/ssd/extra_vars.yaml).
 
 ### Enable protocol handlers
@@ -56,9 +52,9 @@ It will download KoP and MoP from the given URLs. Then, the configuration templa
 You can change the configuration files and then restart the cluster by executing the following command.
 
 ```bash
-TF_STATE=. ansible-playbook \
+ansible-playbook \
   --user ec2-user \
-  --inventory `which terraform-inventory` \
+  --inventory terraform.yaml \
   -e @extra_vars.yaml \
   restart-brokers.yaml
 ```

driver-pulsar/deploy/ssd/deploy.yaml

@@ -26,10 +26,9 @@
         pulsar_version: "{{ pulsar_version | default('2.11.0') }}"
         node_exporter_version: "{{ node_exporter_version | default('1.2.2') }}"
         prometheus_version: "{{ prometheus_version | default('2.31.1') }}"
-        disk_dev: "{{ disk_dev | default(['/dev/nvme1n1', '/dev/nvme2n1']) }}"
     - set_fact:
         pulsar_binary:
-          src: "https://downloads.apache.org/pulsar/pulsar-{{ pulsar_version }}/apache-pulsar-{{ pulsar_version }}-bin.tar.gz"
+          src: "https://archive.apache.org/dist/pulsar/pulsar-{{ pulsar_version }}/apache-pulsar-{{ pulsar_version }}-bin.tar.gz"
           remote: yes
       when: pulsar_binary is not defined
     - set_fact:
@@ -48,13 +47,27 @@
   connection: ssh
   become: true
   tasks:
+    - name: Initialize empty list for devices
+      set_fact:
+        storage_devices: []
+      no_log: true
+
+    - name: Get NVMe devices
+      set_fact:
+        storage_devices: "{{ storage_devices + ['/dev/' ~ item.key ] }}"
+      with_dict: "{{ ansible_devices }}"
+      when: "item.key.startswith('nvme') and not item.value.partitions"
+
+    - name: Show device names
+      debug: var=storage_devices
+
     - name: Format disks
       filesystem:
         fstype: xfs
         dev: '{{ item }}'
       with_items:
-        - "{{ disk_dev[0] }}"
-        - "{{ disk_dev[1] }}"
+        - "{{ storage_devices[0] }}"
+        - "{{ storage_devices[1] }}"
     - name: Mount disks
       mount:
         path: "{{ item.path }}"
@@ -63,21 +76,35 @@
         opts: defaults,noatime,nodiscard
         state: mounted
       with_items:
-        - { path: "/mnt/zookeeper/logs", src: "{{ disk_dev[0] }}" }
-        - { path: "/mnt/zookeeper/data", src: "{{ disk_dev[1] }}" }
+        - { path: "/mnt/zookeeper/logs", src: "{{ storage_devices[0] }}" }
+        - { path: "/mnt/zookeeper/data", src: "{{ storage_devices[1] }}" }
 
 - name: Format and mount disks for Pulsar/BookKeeper hosts
   hosts: pulsar
   connection: ssh
   become: true
   tasks:
+    - name: Initialize empty list for devices
+      set_fact:
+        storage_devices: []
+      no_log: true
+
+    - name: Get NVMe devices
+      set_fact:
+        storage_devices: "{{ storage_devices + ['/dev/' ~ item.key ] }}"
+      with_dict: "{{ ansible_devices }}"
+      when: "item.key.startswith('nvme') and not item.value.partitions"
+
+    - name: Show device names
+      debug: var=storage_devices
+
     - name: Format disks
       filesystem:
          fstype: xfs
          dev: '{{ item }}'
       with_items:
-        - "{{ disk_dev[0] }}"
-        - "{{ disk_dev[1] }}"
+        - "{{ storage_devices[0] }}"
+        - "{{ storage_devices[1] }}"
     - name: Mount disks
       mount:
         path: "{{ item.path }}"
@@ -86,8 +113,8 @@
         opts: defaults,noatime,nodiscard
         state: mounted
       with_items:
-        - { path: "/mnt/journal", src: "{{ disk_dev[0] }}" }
-        - { path: "/mnt/storage", src: "{{ disk_dev[1] }}" }
+        - { path: "/mnt/journal", src: "{{ storage_devices[0] }}" }
+        - { path: "/mnt/storage", src: "{{ storage_devices[1] }}" }
 
 - name: Install Node exporter on Brokers to collect system metrics
   hosts: pulsar
@@ -152,19 +179,39 @@
   connection: ssh
   become: true
   tasks:
+    - name: Install Tuned packages
+      ansible.builtin.package:
+        state: latest
+        update_cache: true
+        name:
+          - tuned
+    - name: Enable Tuned
+      service: name=tuned state=started enabled=yes
     - name: Set performance profile
       command: tuned-adm profile latency-performance
-    - name: Install RPM packages
-      yum:
+    - name: Install packages
+      ansible.builtin.package:
         state: latest
-        pkg:
+        name:
           - wget
-          - java-17-openjdk
-          - java-17-openjdk-devel
           - sysstat
           - vim
           - chrony
-      when: ansible_facts['distribution'] == 'RedHat'
+    - name: Install java on RedHat/CentOS
+      ansible.builtin.package:
+        state: latest
+        name:
+          - java-17-openjdk
+          - java-17-openjdk-devel
+      when:
+        - ansible_facts['distribution'] == 'RedHat' or ansible_facts['distribution'] == 'CentOS'
+    - name: Install java on Debain/Ubuntu
+      ansible.builtin.package:
+        state: latest
+        name:
+          - openjdk-17-jdk
+      when:
+        - ansible_facts['distribution'] == 'Debian' or ansible_facts['distribution'] == 'Ubuntu'
     - file: path=/opt/pulsar state=absent
     - file: path=/opt/pulsar state=directory
     - name: Download Pulsar binary package
@@ -356,12 +403,16 @@
       template:
         src: "templates/client.conf"
         dest: "/opt/pulsar/conf/client.conf"
-    - file: path=/opt/benchmark state=absent
     - name: Copy benchmark code
       unarchive:
         src: ../../../package/target/openmessaging-benchmark-0.0.1-SNAPSHOT-bin.tar.gz
         dest: /opt
-    - shell: mv /opt/openmessaging-benchmark-0.0.1-SNAPSHOT /opt/benchmark
+        creates: /opt/benchmark
+
+    - name: Rename benchmark directory
+      ansible.builtin.shell:
+        cmd: mv /opt/openmessaging-benchmark-0.0.1-SNAPSHOT /opt/benchmark
+        creates: /opt/benchmark
 
     - template:
         src: "templates/workers.yaml"
@@ -436,27 +487,9 @@
   connection: ssh
   become: true
   tasks:
-    - name: Add Extras Repo
-      shell: yum-config-manager --enable rhui-REGION-rhel-server-extras
-      when:
-        - ansible_facts['distribution'] == 'RedHat'
-        - ansible_facts['distribution_major_version'] | int <= 7
-    - name: Docker repo
-      yum_repository:
-        name: docker
-        description: repo for docker
-        baseurl: "https://download.docker.com/linux/centos/{{ ansible_facts['distribution_major_version'] }}/x86_64/stable/"
-        gpgcheck: no
-      when: ansible_facts['distribution'] == 'RedHat'
-    - name: Installing docker
-      yum:
-        state: latest
-        pkg: ['docker-ce']
-    - name: Start docker
-      service:
-        name: docker
-        state: started
-        enabled: yes
+    - name: Install docker
+      include_role:
+        name: geerlingguy.docker
     - file: path=/opt/prometheus state=absent
     - file: path=/opt/prometheus state=directory
     - name: Download Prometheus Binary Package