Merge branch 'rocky' into bump-kolla-release

etc/kayobe/ansible/config-dump.yml

@@ -0,0 +1,6 @@
+---
+
+- name: dump config
+  hosts: localhost
+  roles:
+    - name: kayobe-config-dump

etc/kayobe/ansible/readme.md

@@ -11,3 +11,16 @@ Firewall
 --------
 
 Please see firewall.readme.md.
+
+
+Config Dump
+--------------
+
+To make an archive of overcloud configuration, you can use:
+
+`kayobe playbook run $PWD/etc/kayobe/ansible/config-dump.yml --vault-password-file ~/vaultpassword`
+
+This takes care to redact any passwords from the archive.
+
+You will find the output in `$PWD/redacted-config`.
+

etc/kayobe/ansible/roles/kayobe-config-dump/defaults/main.yml

@@ -0,0 +1,8 @@
+config_dump_path: /tmp/kayobe-config-dump
+output_dir: "{{ lookup('env', 'PWD') ~ '/redacted-config' }}"
+# You can't use ANSIBLE_VAULT_PASSWORD_FILE as you will get:
+# ERROR! The vault-ids default,default are available to encrypt.
+# Specify the vault-id to encrypt with --encrypt-vault-id
+vault_password_file: ~/vaultpassword
+dummy_password: password
+user: stack

etc/kayobe/ansible/roles/kayobe-config-dump/filter_plugins/filters.py

@@ -0,0 +1,34 @@
+import yaml
+
+class FilterModule(object):
+    def filters(self):
+        return {
+            'redact': self.redact,
+        }
+
+    def redact(self, input):
+        data = yaml.load(input)
+        return "---\n" + yaml.dump(redact(data), default_flow_style=False)
+
+def redact_int(_):
+    return 1234
+
+def redact_float(_):
+    return 1.234
+
+def redact_str(_):
+    return "redacted"
+
+def redact_list(xs):
+    return [redact(x) for x in xs]
+
+def redact_dict(x):
+    # keys are assumed to not be secret
+    return {key:redact(value) for key,value in x.items()}
+
+def redact_NoneType(x):
+    return
+
+def redact(x):
+    type_ = type(x).__name__
+    return globals()["redact_%s" % type_](x)

etc/kayobe/ansible/roles/kayobe-config-dump/tasks/main.yml

@@ -0,0 +1,65 @@
+---
+
+- name: create temporary directory
+  tempfile:
+    state: directory
+    suffix: kayobe_config
+  register: tempfile_output
+- block:
+    - name: make a copy of the config
+      synchronize:
+        src: "{{ original_config_path }}"
+        dest: "{{ path }}"
+        recursive: yes
+      delegate_to: "{{ inventory_hostname }}"
+    - name: make a temp vault password file
+      copy:
+        content: "{{ dummy_password }}"
+        dest: "{{ temp_vault_password_file }}"
+    - include_tasks: 'redact.yml'
+      vars:
+        redact_path: "{{ item }}"
+      with_items:
+        - "{{ password_yml_path }}"
+        - "{{ secrets_yml_path }}"
+    - name: generate config
+      command: >-
+        kayobe overcloud service configuration generate
+        --node-config-dir "{{ config_dump_path }}"
+        --extra-vars "ansible_user={{ user }}"
+        --kolla-extra-vars "ansible_user={{ user }}"
+      environment: "{{ env }}"
+      register: result
+    - file:
+        path: "{{ output_dir }}"
+        state: directory
+    - name: save kayobe config
+      command: >-
+        kayobe overcloud service configuration save
+        --node-config-dir "{{ config_dump_path }}"
+        --output-dir "{{ output_dir }}"
+        --extra-vars "ansible_user={{ user }}"
+      environment: "{{ env }}"
+      register: result
+  vars:
+    original_config_path: "{{ lookup('env', 'KAYOBE_CONFIG_PATH') ~ '/../..' }}"
+    temp_vault_password_file: "{{ path ~ '/vault.pass' }}"
+    path: "{{ tempfile_output.path }}"
+    password_yml_path: "{{ path ~ '/etc/kayobe/kolla/passwords.yml' }}"
+    secrets_yml_path: "{{ path ~ '/etc/kayobe/secrets.yml' }}"
+    env:
+      KAYOBE_CONFIG_PATH: "{{ path ~ '/etc/kayobe' }}"
+      KAYOBE_VAULT_PASSWORD: "{{ dummy_password }}"
+      KAYOBE_BASE_PATH: "{{ path }}"
+  rescue:
+    - copy:
+        content: "{{ result.stdout }}"
+        dest: /tmp/kayobe-config-dump-debug
+      when: result | default(False) and result.rc != 0
+  always:
+    - debug:
+        var: tempfile_output
+    - name: remove tempdir
+      file:
+        path: "{{ tempfile_output.path }}"
+        state: absent

etc/kayobe/ansible/roles/kayobe-config-dump/tasks/redact.yml

@@ -0,0 +1,13 @@
+---
+
+- name: redact passwords
+  copy:
+    content: "{{ plaintext | redact }}"
+    dest: "{{ redact_path }}"
+    mode: 0600
+  vars:
+    plaintext: "{{ lookup('pipe', 'ansible-vault view --vault-id {{ vault_password_file }} ' + redact_path) }}"
+
+- name: reencrypt
+  command: >-
+    ansible-vault encrypt --vault-id "{{ temp_vault_password_file }}" --encrypt-vault-id default "{{ redact_path }}"

etc/kayobe/inventory/group_vars/seed/firewall

@@ -8,11 +8,19 @@ firewallgen_ipv4_input_allow_rules:
     proto: tcp
     destination: "127.0.0.1"
     comment: "hint: used by 'master'"
+  - interface: "lo"
+    port: 53
+    proto: tcp
+    comment: "hint: used by 'dnsmasq' in docker container 'bifrost_deploy'"
   - interface: "lo"
     port: 53
     proto: tcp
     destination: "127.0.0.1"
     comment: "hint: used by 'dnsmasq' in docker container 'bifrost_deploy'"
+  - interface: "lo"
+    port: 3306
+    proto: tcp
+    comment: "hint: used by 'mysqld' in docker container 'bifrost_deploy'"
   - interface: "lo"
     port: 4369
     proto: tcp
@@ -21,6 +29,10 @@ firewallgen_ipv4_input_allow_rules:
     port: 5050
     proto: tcp
     comment: "hint: used by 'ironic-inspecto' in docker container 'bifrost_deploy'"
+  - interface: "lo"
+    port: 5672
+    proto: tcp
+    comment: "hint: used by 'beam.smp' in docker container 'bifrost_deploy'"
   - interface: "lo"
     port: 25672
     proto: tcp
@@ -34,16 +46,6 @@ firewallgen_ipv4_input_allow_rules:
     proto: tcp
     destination: "{{ ilab_net_name | net_ip }}"
     comment: "SSH via ILAB"
-  - interface: "{{ internal_net_name | net_interface }}"
-    port: 9197
-    proto: tcp
-    destination: "{{ internal_net_name | net_ip }}"
-    comment: "mtail"
-  - interface: "{{ internal_net_name | net_interface }}"
-    port: 18080
-    proto: tcp
-    destination: "{{ internal_net_name | net_ip }}"
-    comment: "cadvisor"
   - interface: "{{ provision_oc_net_interface }}"
     port: 53
     proto: tcp
@@ -57,6 +59,10 @@ firewallgen_ipv4_input_allow_rules:
     port: 8080
     proto: tcp
     comment: "hint: used by 'nginx' in docker container 'bifrost_deploy'"
+  - interface: "{{ provision_oc_net_name | net_interface }}"
+    port: 4000
+    proto: tcp
+    comment: "hint: used by 'docker-proxy'"
   - interface: "docker0"
     port: 123
     proto: udp

etc/kayobe/inventory/group_vars/seed/firewall-extra

@@ -0,0 +1,11 @@
+---
+
+firewallgen_ipv4_forward_allow:
+  - in-interface: "{{ ilab_net_interface }}"
+    out-interface: "{{ admin_oc_net_name | net_interface }}"
+    state: RELATED,ESTABLISHED
+  - in-interface: "{{ admin_oc_net_name | net_interface }}"
+    out-interface: "{{ ilab_net_interface }}"
+
+firewallgen_ipv4_snat:
+  - out-interface: "{{ ilab_net_interface }}"

etc/kayobe/inventory/group_vars/seed/firewallgen

@@ -19,15 +19,20 @@ firewallgen_ipv4_input_allow_rewrite_rules_extra:
   - '. |= map( if .port == 4369 and .proto == "tcp"  then .interface = "lo" else . end)'
 # rabbit (doesn't seem to referenced in any config file)
   - '. |= map( if .port == 25672 and .proto == "tcp"  then .interface = "lo" else . end)'
+# bifrost mysql
+  - '. |= map( if .port == 3306 and .proto == "tcp"  then .interface = "lo" else . end)'
+# bifrost rabbitmq
+  - '. |= map( if .port == 5672 and .proto == "tcp"  then .interface = "lo" else . end)'
+# DNS ?
+  - '. |= map( if .port == 53 then .interface = "lo" else . end)'
+# docker-proxy for docker regsitry
+  - >-
+    . |= map( if .port == 4000 and .proto == "tcp" then
+            .interface = "{{ firewallgen_interface_tmpl % 'provision_oc_net_name' }}" else . end)
 
 firewallgen_ipv4_input_allow_custom_rules_extra:
-  # These are ipv4-mapped ipv6 addresses and don't show in ss -nlpt -4 output
-  - interface: "{{ firewallgen_interface_tmpl % 'provision_oc_net_name' }}"
-    port: 5000
-    proto: tcp
-    destination: "{% raw %}{{ provision_oc_net_name | net_ip }}{% endraw %}"
-  - interface: "{{ firewallgen_interface_tmpl % 'ilab_net_name' }}"
-    port: 22
-    proto: tcp
-    destination: "{% raw %}{{ ilab_net_name | net_ip }}{% endraw %}"
-    comment: SSH via ILAB
+ - interface: "{{ firewallgen_interface_tmpl % 'ilab_net_name' }}"
+   port: 22
+   proto: tcp
+   destination: "{% raw %}{{ ilab_net_name | net_ip }}{% endraw %}"
+   comment: SSH via ILAB