Merge remote-tracking branch 'origin/develop' into rocky-develop

.gitignore

@@ -20,3 +20,4 @@ __pycache__/
 
 /test.*
 /callback_plugins/clean.py
+.vscode/

group_vars/all.yml

@@ -359,9 +359,15 @@ aws_security_group_persistant_rules_ssh_proxy:
       - "22"
     group_name: "{{ aws_security_group_app_name }}"
 
+aws_security_group_persistent_rules_tinyproxy:
+  - proto: tcp
+    ports:
+      - "{{ tinyproxy_port }}"
+    group_name: "{{ aws_security_group_app_name }}"
 
 aws_security_group_persistant_rules_tpl: [
-  "{{ mageops_ssh_proxy_persistant | ternary(aws_security_group_persistant_rules_ssh_proxy, []) }}"
+  "{{ mageops_ssh_proxy_persistant | ternary(aws_security_group_persistant_rules_ssh_proxy, []) }}",
+  "{{ mageops_tinyproxy_persistant_enabled | ternary(aws_security_group_persistent_rules_tinyproxy, []) }}"
 ]
 
 aws_security_group_persistant_rules: "{{ aws_security_group_persistant_rules_tpl | flatten }}"
@@ -1503,9 +1509,11 @@ nginx_blacklist_vhost_check_include_file: "{{ nginx_etc_dir }}/magento_url_black
 # --------  Elasticsearch  --------
 # ---------------------------------
 
-# Install elasticsearch 6.x as this is the latest version fully supported by
-# ElasticSuite. There's already 7.x though...
-repo_elasticsearch_variants_enabled: ['6']
+# Install elasticsearch 7.x as this is the latest version fully supported by
+# ElasticSuite. There's already 8.x though...
+repo_elasticsearch_variants_enabled: ['7']
+# mageops_elasticsearch_version: 8.4
+mageops_elasticsearch_version:
 
 elasticsearch_cluster_name: "{{ mageops_app_id }}"
 elasticsearch_node_name: master
@@ -1864,17 +1872,30 @@ packages_remove: "{{ mageops_packages_banned }}"
 
 # Package locks, for more info, check cs.versionlock role
 
+mageops_elasticsearch_version_full: "elasticsearch-{{mageops_elasticsearch_version}}.*"
+versionlock_persistent_packages_base_tpl: [
+  "{{ mageops_elasticsearch_version | ternary( mageops_elasticsearch_version_full, [] ) }}"
+]
+
 # Package locks per instance
 versionlock_varnish_packages:
   - varnish-6.0.*
 versionlock_persistent_packages: []
 versionlock_app_node_packages: []
 
+versionlock_varnish_packages_base: []
+versionlock_persistent_packages_base: "{{ versionlock_persistent_packages_base_tpl | flatten }}"
+versionlock_app_node_packages_base: []
+
 # Package bans per instance
 versionlock_varnish_ban_packages: []
 versionlock_persistent_ban_packages: []
 versionlock_app_node_ban_packages: []
 
+versionlock_varnish_ban_packages_base: []
+versionlock_persistent_ban_packages_base: []
+versionlock_app_node_ban_packages_base: []
+
 # Per project customizations
 versionlock_varnish_packages_extra: []
 versionlock_varnish_ban_packages_extra: []
@@ -1916,6 +1937,22 @@ sshd_config_users_tpl: [
 
 sshd_config_users: "{{ sshd_config_users_tpl | combine }}"
 
+mageops_tinyproxy_persistant_enabled: no
+tinyproxy_port: 8888
+
+tinyproxy_allowed_hosts_default:
+  - 127.0.0.1
+
+tinyproxy_allowed_hosts_aws:
+  - "{{ aws_vpc_subnet_prefix }}.0.0/16"
+
+tinyproxy_allowed_hosts_tpl: [
+  "{{ tinyproxy_allowed_hosts_default }}",
+  "{{ aws_use | ternary(tinyproxy_allowed_hosts_aws, []) }}"
+]
+tinyproxy_allowed_hosts: "{{ tinyproxy_allowed_hosts_tpl | flatten }}"
+
+
 # ---------------------------
 # --------  Helpers  --------
 # ---------------------------

roles/cs.aws-iam-employee-access/templates/employee_full_access.policy.json

@@ -17,7 +17,8 @@
                 "sns:ListTopics",
                 "logs:DescribeLogStreams",
                 "logs:GetLogEvents",
-                "elasticfilesystem:*"
+                "elasticfilesystem:*",
+                "license-manager:*"
             ],
             "Effect": "Allow",
             "Resource": "*"

roles/cs.aws-rds/defaults/main.yml

@@ -3,7 +3,7 @@ aws_rds_backup_window: 02:00-04:00
 aws_rds_maintenance_window: Mon:04:00-Mon:05:00
 aws_rds_auto_minor_version_upgrade: yes
 aws_rds_allow_major_version_upgrade: no
-aws_rds_storage_size: 10
+aws_rds_storage_size: 20
 aws_rds_storage_encrypt: no
 aws_rds_dedicated_encryption_key: no
 aws_rds_db_engine: mariadb

roles/cs.elasticsearch/tasks/main.yml

@@ -4,12 +4,6 @@
     state: latest
   register: elasticsearch_packages_install
 
-# FIXME: Remove me after May 2022
-- name: Remove manualy applied log4j workaround (now included in generated template)
-  file:
-    path: /etc/elasticsearch/jvm.options.d/workaround-log4j.options
-    state: absent
-
 # In case `/var` or `/var/run` is a tmpfs (common thing nowadays)
 - name: Ensure var directories always exist
   lineinfile:
@@ -90,21 +84,6 @@
     msg: |
       ERROR! Invalid, undetected or unsupported elasticsearch version!
 
-- name: Configure elasticsearch 5
-  template:
-    src: "{{ item.key }}"
-    dest: /etc/elasticsearch/{{ item.value }}
-    owner: root
-    group: elasticsearch
-  with_dict:
-    es5-jvm.options: jvm.options
-    elasticsearch.yml: elasticsearch.yml
-    log4j2.properties: log4j2.properties
-  notify: Restart elasticsearch
-  when: >-
-    elasticsearch_version_number is version('5.0.0', '>=')
-      and elasticsearch_version_number is version('6.0.0', '<')
-
 - name: Configure elasticsearch 6
   template:
     src: "{{ item.key }}"

roles/cs.elasticsearch/templates/elasticsearch.yml

@@ -7,18 +7,41 @@
 
 cluster.name: {{ elasticsearch_cluster_name }}
 {% if elasticsearch_version_number is version('7.0.0', '>=') %}
-cluster.initial_master_nodes: ["{{ elasticsearch_node_name }}"]
+discovery.type: single-node
+{% endif %}
+
+{% if elasticsearch_version_number is version('7.16.0', '>=') %}
+cluster.deprecation_indexing.enabled: false
+xpack.security.enabled: false
 {% endif %}
 
 path.data: /var/lib/elasticsearch
 path.logs: /var/log/elasticsearch
 
 network.host: {{ elasticsearch_network_host }}
 http.port: {{ elasticsearch_http_port }}
+
+{% if elasticsearch_version_number is version('6.7', '>=') %}
+transport.port: {{ elasticsearch_transport_tcp_port }}
+{% else %}
 transport.tcp.port: {{ elasticsearch_transport_tcp_port }}
+{% endif %}
 
 node.name: {{ elasticsearch_node_name }}
-node.data: true
+{% if elasticsearch_version_number is version('7.9.0', '>=') %}
+node.roles: ['data','master']
+{% else %}
 node.master: true
+node.data: true
+{% endif %}
+{% if elasticsearch_version_number is version('8.0.0', '>=') %}
+# Enable or disable encryption for HTTP API client connections, such as Kibana, Logstash, and Agents
+xpack.security.http.ssl:
+  enabled: false
+
+# Enable or disable encryption and mutual authentication between cluster nodes
+xpack.security.transport.ssl:
+  enabled: false
+{% endif %}
 
 bootstrap.memory_lock: {{ elasticsearch_memlock_enable | ternary('true', 'false') }}

roles/cs.elasticsearch/templates/es7-log4j2.properties

@@ -92,7 +92,7 @@ appender.deprecation_rolling_old.strategy.type = DefaultRolloverStrategy
 appender.deprecation_rolling_old.strategy.max = 4
 #################################################
 logger.deprecation.name = org.elasticsearch.deprecation
-logger.deprecation.level = deprecation
+logger.deprecation.level = error
 logger.deprecation.appenderRef.deprecation_rolling.ref = deprecation_rolling
 logger.deprecation.appenderRef.deprecation_rolling_old.ref = deprecation_rolling_old
 logger.deprecation.appenderRef.header_warning.ref = header_warning

roles/cs.mageops-cli/files/libmageops.bash

@@ -75,9 +75,9 @@ mageops::update_s3_file() {
 
 
 mageops::clear_php_opcache() {
-    for die in "${config__opcache_file_paths[@]}";do
+    for dir in "${config__opcache_file_paths[@]}";do
         echo "Clearing opcache $dir..."
-        rm -rf "${config__opcache_file_paths[@]}"
+        rm -rf "$dir"
     done
     systemctl reload php-fpm
 }

roles/cs.new-relic/templates/cron.j2

@@ -1,4 +1,4 @@
 # Enable new relic features
-{{ new_relic_cron_start }} root sh -c 'mageopscli is_cron_node && mageopscli set_feature_flag newrelic_apm true'
+{{ new_relic_cron_start }} root sh -c '/usr/local/bin/mageopscli is_cron_node && /usr/local/bin/mageopscli set_feature_flag newrelic_apm true'
 # Disable new relic features
-{{ new_relic_cron_stop }} root sh -c 'mageopscli is_cron_node && mageopscli set_feature_flag newrelic_apm false'
+{{ new_relic_cron_stop }} root sh -c '/usr/local/bin/mageopscli is_cron_node && /usr/local/bin/mageopscli set_feature_flag newrelic_apm false'

roles/cs.nginx-https-termination/templates/nginx.vhost.conf

@@ -82,14 +82,19 @@ server {
 {% else %}
     location / {
 {% endif %}
-{% for cookie_js_name, cookie_config in https_termination_nginx_server_cookie_rewrite_config.items() %}
-        {%- set cookie_js_bare_var_name = cookie_js_name | lower | regex_replace('[^_a-z0-9]', '_') %}
-        add_header "Set-Cookie" "${MAGEOPS_COOKIE_REWRITE_{{ cookie_js_bare_var_name }}}
-{%- if cookie_config.rewrite_domain|default([])|length > 0 -%}
-        ${MAGEOPS_COOKIE_REWRITE_DOMAIN_{{ cookie_js_bare_var_name }}}
-{%- endif -%}
-        ";
-{% endfor %}
+{% if https_termination_nginx_server_cookie_rewrite_config | length > 0 %}
+    if ($request_uri !~* ^/(media|static)) {
+    {% for cookie_js_name, cookie_config in https_termination_nginx_server_cookie_rewrite_config.items() %}
+            {%- set cookie_js_bare_var_name = cookie_js_name | lower | regex_replace('[^_a-z0-9]', '_') %}
+            add_header "Set-Cookie" "${MAGEOPS_COOKIE_REWRITE_{{ cookie_js_bare_var_name }}}
+    {%- if cookie_config.rewrite_domain|default([])|length > 0 -%}
+            ${MAGEOPS_COOKIE_REWRITE_DOMAIN_{{ cookie_js_bare_var_name }}}
+    {%- endif -%}
+            ";
+    {% endfor %}
+    }
+{% endif %}
+
 {% if vhost.redirect_to %}
         {% if vhost.redirect_to_style == 'deep' %}rewrite ^/(.*)$ $scheme://{{ vhost.redirect_to }}/$1{% endif %}
     {%- if vhost.redirect_to_style == 'absolute' %}rewrite / {{ vhost.redirect_to }}{% endif %}

roles/cs.nginx-magento/defaults/main.yml

@@ -107,3 +107,8 @@ nginx_custom_php_scripts: []
 #   extra_nginx_config: |-
 #     set $PHP_MEMORY_LIMIT 128M;
 #     set $PHP_MAX_EXECUTION_TIME 60;
+
+nginx_magento_custom_location: []
+# - path_pattern: '~ ^/custom_location'
+#   nginx_config: |-
+#    try_files /custom_path/$uri;

roles/cs.nginx-magento/templates/magento_server_body.conf.j2

@@ -311,6 +311,12 @@ location ~ {{ custom.path_pattern }} {
 }
 {% endfor %}
 
+{% for custom in nginx_magento_custom_location %}
+location {{ custom.path_pattern }} {
+    {{ custom.nginx_config }}
+}
+{% endfor %}
+
 location ~ .*\.php$ {
     deny all;
 }