Implement cron serivce

group_vars/all.yml

@@ -41,6 +41,8 @@ aws_iam_role_basic_lambda_execution: "{{ aws_iam_name_prefix }}BasicLambdaExecut
 aws_iam_role_autoscaling_event_lambda: "{{ aws_iam_name_prefix }}HandleAutoscalingEventLambdaExecution"
 aws_iam_role_update_varnish_backend_lambda_execution: "{{ aws_iam_name_prefix }}UpdateVarnishBackendsLambdaExecution"
 aws_iam_role_import_scaling_lambda_execution: "{{ aws_iam_name_prefix }}HandleImportScalingLambdaExecution"
+aws_iam_policy_node_coordinator_lambda: "{{ aws_iam_name_prefix }}HandleNodeCoordinatorLambda"
+aws_iam_role_node_coordinator_lambda_execution: "{{ aws_iam_name_prefix }}HandleNodeCoordinatorLambdaExecution"
 
 # ----------------------------
 # --------  AWS Tags  --------
@@ -406,7 +408,7 @@ aws_s3_bucket_policies: []
 # --------  AWS EFS  --------
 # --------------------------
 
-# The main EFS root directory mountpoin on each node 
+# The main EFS root directory mountpoin on each node
 # Note: Not used directly by app bu as helper during provisioning and conveience for users
 efs_root_mountpoint: /mnt/efs
 
@@ -705,6 +707,8 @@ magento_cron_user: "{{ mageops_app_id }}"
 # magento_cron_password:
 
 magento_cron_disabled: no
+magento_cron_http: "{{ not aws_use }}"
+aws_magento_cron_enabled: "{{ aws_use }}"
 magento_cron_observer_relative_path: vendor/magento/module-cron/Observer/ProcessCronQueueObserver.php
 
 
@@ -1524,4 +1528,9 @@ ntp_timezone: "{{ mageops_timezone }}"
 # -------------------------------------------------
 
 mageops_notifications_slack_channel: ~
-mageops_notifications_slack_enable: "{{ mageops_notifications_slack_channel | default('') | length > 0 }}" 
+mageops_notifications_slack_enable: "{{ mageops_notifications_slack_channel | default('') | length > 0 }}"
+
+
+# Misc
+aws_autoscaling_triggers_list: []
+aws_autoscaling_lambda_functions_list: []

roles/cs.aws-autoscaling-triggers/defaults/main.yml

@@ -0,0 +1,21 @@
+aws_autoscaling_event_pattern:
+    source:
+        - aws.autoscaling
+    detail-type:
+        - "EC2 Instance Launch Successful"
+        - "EC2 Instance-terminate Lifecycle Action"
+        - "EC2 Instance Terminate Successful"
+
+# Those we don't handle for now
+# XXX(Szpadel): Why we cannot handle those?
+#        - "EC2 Instance-launch Lifecycle Action"
+#        - "EC2 Instance Terminate Successful"
+#        - "EC2 Instance Launch Unsuccessful"
+#        - "EC2 Instance Terminate Unsuccessful"
+
+
+aws_autoscaling_triggers_list: []
+# - id:
+#   arn:
+
+aws_autoscaling_lambda_functions_list: []

roles/cs.aws-autoscaling-triggers/tasks/main.yml

@@ -0,0 +1,20 @@
+- name: Setup autoscaling event trigger
+  cloudwatchevent_rule:
+    name: "trigger-autoscaling-lambda-{{ mageops_app_name }}"
+    region: "{{ aws_region }}"
+    state: present
+    event_pattern: "{{ aws_autoscaling_event_pattern | to_json }}"
+    targets: "{{ aws_autoscaling_triggers_list }}"
+
+  register: aws_autoscaling_event
+
+- name: Allow autoscaling event handler lambda to be executed by CloudWatch Events
+  lambda_policy:
+    state: present
+    region: "{{ aws_region }}"
+    function_name: "{{ item }}"
+    action: "lambda:InvokeFunction"
+    principal: "events.amazonaws.com"
+    statement_id: "AWSEvents-handleAutoscalingEvent-{{ mageops_app_name }}"
+    source_arn: "{{ aws_autoscaling_event.rule.arn }}"
+  with_items: "{{ aws_autoscaling_lambda_functions_list }}"

roles/cs.aws-autoscaling/defaults/main.yml

@@ -12,4 +12,4 @@ autoscaling_asg_tags: {}
 
 
 # This didn't work with previous ansible versions
-# autoscaling_asg_replace_batch_wait:
+# autoscaling_asg_replace_batch_wait:

roles/cs.aws-iam/defaults/main.yml

@@ -16,12 +16,14 @@ aws_iam_group_standard_policies: "{{ aws_iam_name_prefix }}StandardPolicies"
 aws_iam_policy_autoscaling_event_lambda: "{{ aws_iam_name_prefix }}HandleAutoscalingEventLambda"
 aws_iam_policy_update_varnish_backends: "{{ aws_iam_name_prefix }}UpdateVarnishBackendsLambda"
 aws_iam_policy_import_scaling_lambda: "{{ aws_iam_name_prefix }}HandleImportScalingLambda"
+aws_iam_policy_node_coordinator_lambda: "{{ aws_iam_name_prefix }}HandleNodeCoordinatorLambda"
 
 aws_iam_role_basic_lambda_execution: "{{ aws_iam_name_prefix }}BasicLambdaExecution"
 aws_iam_role_autoscaling_event_lambda: "{{ aws_iam_name_prefix }}HandleAutoscalingEventLambdaExecution"
 aws_iam_role_update_varnish_backend_lambda_execution: "{{ aws_iam_name_prefix }}UpdateVarnishBackendsLambdaExecution"
 aws_iam_role_import_scaling_lambda_execution: "{{ aws_iam_name_prefix }}HandleImportScalingLambdaExecution"
+aws_iam_role_node_coordinator_lambda_execution: "{{ aws_iam_name_prefix }}HandleNodeCoordinatorLambdaExecution"
 
 aws_iam_role_app_node: "{{ aws_iam_name_prefix }}AppNode"
 aws_iam_role_varnish: "{{ aws_iam_role_app_node }}"
-aws_iam_role_persistent_node: "{{ aws_iam_name_prefix }}PersistentNode"
+aws_iam_role_persistent_node: "{{ aws_iam_name_prefix }}PersistentNode"

roles/cs.aws-iam/tasks/lambda-roles.yml

@@ -6,6 +6,8 @@
     managed_policy:
       - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole
 
+# Autoscale event
+
 - name: Create custom policy for autoscale events handler lambda
   iam_managed_policy:
     policy_name: "{{ aws_iam_policy_autoscaling_event_lambda }}"
@@ -23,6 +25,8 @@
       - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole
       - "{{ iam_handle_autoscaling_event_lambda.policy.arn }}"
 
+# update backends
+
 - name: Create custom policy for update varnish backends lambda
   iam_managed_policy:
     policy_name: "{{ aws_iam_policy_update_varnish_backends }}"
@@ -31,6 +35,17 @@
     state: present
   register: iam_update_varnish_backends_lambda
 
+- name: Create role for handle autoscaling event lambda
+  iam_role:
+    assume_role_policy_document: "{{ lookup('template', 'lambda_execution.policy.json') }}"
+    name: "{{ aws_iam_role_update_varnish_backend_lambda_execution }}"
+    state: present
+    managed_policy:
+      - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole
+      - "{{ iam_update_varnish_backends_lambda.policy.arn }}"
+
+# import scaling
+
 - name: Create custom policy for import scaling handler lambda
   iam_managed_policy:
     policy_name: "{{ aws_iam_policy_import_scaling_lambda }}"
@@ -39,20 +54,30 @@
     state: present
   register: iam_import_scaling_handler_lambda
 
-- name: Create role for handle autoscaling event lambda
+- name: Create role for import scaling handler lambda
   iam_role:
     assume_role_policy_document: "{{ lookup('template', 'lambda_execution.policy.json') }}"
-    name: "{{ aws_iam_role_update_varnish_backend_lambda_execution }}"
+    name: "{{ aws_iam_role_import_scaling_lambda_execution }}"
     state: present
     managed_policy:
       - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole
-      - "{{ iam_update_varnish_backends_lambda.policy.arn }}"
+      - "{{ iam_import_scaling_handler_lambda.policy.arn }}"
 
-- name: Create role for import scaling handler lambda
+# node coordinator
+
+- name: Create custom policy for node coordinator handler lambda
+  iam_managed_policy:
+    policy_name: "{{ aws_iam_policy_node_coordinator_lambda }}"
+    policy_description: Custom permissions for node coordinator lambda
+    policy: "{{ lookup('template', 'handle_node_coordinator_lambda.policy.json') }}"
+    state: present
+  register: iam_node_coordinator_handler_lambda
+
+- name: Create role for node coordinator handler lambda
   iam_role:
     assume_role_policy_document: "{{ lookup('template', 'lambda_execution.policy.json') }}"
-    name: "{{ aws_iam_role_import_scaling_lambda_execution }}"
+    name: "{{ aws_iam_role_node_coordinator_lambda_execution }}"
     state: present
     managed_policy:
       - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole
-      - "{{ iam_import_scaling_handler_lambda.policy.arn }}"
+      - "{{ iam_node_coordinator_handler_lambda.policy.arn }}"

roles/cs.aws-iam/templates/handle_autoscaling_event_lambda.policy.json

@@ -21,8 +21,9 @@
             ],
             "Resource": [
                 "arn:aws:lambda:{{ aws_region }}:{{ aws_account_id }}:function:updateVarnishBackends*",
+                "arn:aws:lambda:{{ aws_region }}:{{ aws_account_id }}:function:nodeCoordinator*",
                 "arn:aws:autoscaling:*:*:autoScalingGroup:*:autoScalingGroupName/*"
             ]
         }
     ]
-}
+}

roles/cs.aws-iam/templates/handle_node_coordinator_lambda.policy.json

@@ -0,0 +1,14 @@
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Action": [
+                "ec2:DescribeInstances",
+                "ec2:DeleteTags",
+                "ec2:CreateTags"
+            ],
+            "Resource": "*"
+        }
+    ]
+}

roles/cs.aws-lambda-node-coordinator/defaults/main.yml

@@ -0,0 +1,4 @@
+aws_lambda_node_coordinator_package_url: "https://github.com/mageops/aws-lambda-node-coordinator/releases/download/v0.1.4/lambda.zip"
+aws_lambda_node_coordinator_package_path: "{{ mageops_ansible_temp_dir }}/node_coordinator_lambda_v0.1.4.zip"
+aws_lambda_node_coordinator_runtime: nodejs12.x
+aws_lambda_handle_node_coordinator_autoscaling_event_name: "handleNodeCoordinatorAutoscalingEvent-{{ mageops_app_name }}"

roles/cs.aws-lambda-node-coordinator/meta/main.yml

@@ -0,0 +1,2 @@
+dependencies:
+  - role: cs.ansible-plugins

roles/cs.aws-lambda-node-coordinator/tasks/main.yml

@@ -0,0 +1,34 @@
+- name: Download lambda deploy packages
+  get_url:
+    url: "{{ aws_lambda_node_coordinator_package_url }}"
+    dest: "{{ aws_lambda_node_coordinator_package_path }}"
+
+- name: Register lambda handler for coordinating nodes
+  lambda:
+    name: "{{ aws_lambda_handle_node_coordinator_autoscaling_event_name }}"
+    state: present
+    zip_file: "{{ aws_lambda_node_coordinator_package_path }}"
+    runtime: "{{ aws_lambda_node_coordinator_runtime }}"
+    role: "arn:aws:iam::{{ aws_account_id }}:role/{{ aws_iam_role_node_coordinator_lambda_execution }}"
+    handler: index.handler
+    region: "{{ aws_region }}"
+    timeout: 15
+    environment_variables:
+      ENVIRONMENT: "{{ mageops_environment }}"
+      PROJECT: "{{ mageops_project }}"
+  register: handle_autoscaling_event_lambda
+
+- name: Register autoscaling trigger
+  set_fact:
+    coordinator_autoscaling_trigger:
+      id: "{{ aws_lambda_handle_node_coordinator_autoscaling_event_name }}-target"
+      arn: "{{ handle_autoscaling_event_lambda.configuration.function_arn }}"
+
+- name: Append trigger to list
+  set_fact:
+    aws_autoscaling_triggers_list: "{{ aws_autoscaling_triggers_list + [coordinator_autoscaling_trigger] }}"
+    aws_autoscaling_lambda_functions_list: "{{ aws_autoscaling_lambda_functions_list + [aws_lambda_handle_node_coordinator_autoscaling_event_name] }}"
+
+- name: Require cs.aws-autoscaling-triggers finalizer
+  set_fact:
+    finalize_roles: "{{ finalize_roles + ['cs.aws-autoscaling-triggers'] }}"

roles/cs.aws-lambda-varnish/defaults/main.yml

@@ -2,23 +2,9 @@ aws_lambda_varnish_deploy_package_url: "https://github.com/mageops/aws-lambdas-a
 aws_lambda_varnish_deploy_package_path: "{{ mageops_ansible_temp_dir }}/varnish-lambda-deploy-package.zip"
 aws_lambda_varnish_runtime: python3.7
 
-aws_lambda_handle_autoscaling_event_name: "handleAutoscalingEvent-{{ mageops_app_name }}"
+aws_lambda_handle_varnish_autoscaling_event_name: "handleVarnishAutoscalingEvent-{{ mageops_app_name }}"
 aws_lambda_update_varnish_backends_name: "updateVarnishBackends-{{ mageops_app_name }}"
 
 
 aws_lambda_update_varnish_vcl_function_s3_bucket_name: "{{ aws_s3_secret_bucket }}"
 aws_lambda_update_varnish_vcl_function_ssh_key_name: "{{ varnish_manager_ssh_key_filename }}"
-aws_lambda_update_varnish_vcl_autoscaling_event_pattern:
-    source:
-        - aws.autoscaling
-    detail-type:
-        - "EC2 Instance Launch Successful"
-        - "EC2 Instance-terminate Lifecycle Action"
-
-
-# Those we don't handle for now
-#        - "EC2 Instance-launch Lifecycle Action"
-#        - "EC2 Instance Terminate Successful"
-#        - "EC2 Instance Launch Unsuccessful"
-#        - "EC2 Instance Terminate Unsuccessful"
-

roles/cs.aws-lambda-varnish/tasks/create-autoscaling-handler-lambda.yml

@@ -8,7 +8,7 @@
 
 - name: Register lambda handler for autoscaling event
   lambda:
-    name: "{{ aws_lambda_handle_autoscaling_event_name }}"
+    name: "{{ aws_lambda_handle_varnish_autoscaling_event_name }}"
     state: present
     zip_file: "{{ aws_lambda_varnish_deploy_package_path }}"
     runtime: "{{ aws_lambda_varnish_runtime }}"
@@ -28,24 +28,18 @@
         UPDATE_LAMBDA_NAME: "{{ aws_lambda_update_varnish_backends_name }}"
   register: handle_autoscaling_event_lambda
 
-- name: Setup autoscaling event trigger
-  cloudwatchevent_rule:
-    name: "trigger-autoscaling-lambda-{{ mageops_app_name }}"
-    region: "{{ aws_region }}"
-    state: present
-    event_pattern: "{{ aws_lambda_update_varnish_vcl_autoscaling_event_pattern | to_json }}"
-    targets:
-        - id: "{{ aws_lambda_handle_autoscaling_event_name }}-target"
-          arn: "{{ handle_autoscaling_event_lambda.configuration.function_arn }}"
-  register: varnish_autoscaling_event
+- name: Register autoscaling trigger
+  set_fact:
+    varnish_autoscaling_trigger:
+      id: "{{ aws_lambda_handle_varnish_autoscaling_event_name }}-target"
+      arn: "{{ handle_autoscaling_event_lambda.configuration.function_arn }}"
 
-- name: Allow autoscaling event handler lambda to be executed by CloudWatch Events
-  lambda_policy:
-    state: present
-    region: "{{ aws_region }}"
-    function_name: "{{ aws_lambda_handle_autoscaling_event_name }}"
-    action: "lambda:InvokeFunction"
-    principal: "events.amazonaws.com"
-    statement_id: "AWSEvents-handleAutoscalingEvent-{{ mageops_app_name }}"
-    source_arn: "{{ varnish_autoscaling_event.rule.arn }}"
+- name: Append trigger to list
+  set_fact:
+    aws_autoscaling_triggers_list: "{{ aws_autoscaling_triggers_list + [varnish_autoscaling_trigger] }}"
+    aws_autoscaling_lambda_functions_list: "{{ aws_autoscaling_lambda_functions_list + [aws_lambda_handle_varnish_autoscaling_event_name] }}"
 
+
+- name: Require cs.aws-autoscaling-triggers finalizer
+  set_fact:
+    finalize_roles: "{{ finalize_roles + ['cs.aws-autoscaling-triggers'] }}"

roles/cs.aws-lambda-varnish/tasks/create-update-varnish-backends-lambda.yml

@@ -22,8 +22,3 @@
     environment_variables:
       BACKEND_TEMPLATE: "{{ lookup('file', 'roles/cs.varnish/templates/vcl/backends.vcl.j2')|to_json }}"
       BACKEND_TEMPLATE_VARS: "{{ varnish_backend_template_vars | to_json }}"
-
-
-
-
-

roles/cs.aws-magento-cron/defaults/main.yml

@@ -0,0 +1,8 @@
+# Should this service be enabled
+aws_magento_cron_enabled: no
+
+aws_magento_cron_files_install:
+  - { src: "aws-cron-service.sh", dest: "/usr/local/libexec/aws-cron-service.sh", mode: "0755" }
+
+aws_magento_cron_files_install_templated:
+  - { src: "magento-cron-aws.service.j2", dest: "/etc/systemd/system/magento-cron-aws.service" }

roles/cs.aws-magento-cron/files/aws-cron-service.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+set -e
+
+get_current_instance_id() {
+    curl -Lsf http://instance-data/latest/meta-data/instance-id
+}
+
+get_current_region() {
+    curl -Lsf http://instance-data/latest/meta-data/placement/availability-zone | sed 's/[a-z]$//'
+}
+
+get_ec2_tag_value() {
+    local region=$1
+    local instance_id=$2
+    local tag_name=$3
+
+    aws ec2 describe-tags --region "${region}" --filters "Name=resource-id,Values=${instance_id}" "Name=key,Values=${tag_name}" | jq -r '.Tags[].Value'
+}
+
+is_tag_exists() {
+    local tag_name=$1
+    local tag_value=$2
+
+    local instance_id
+    local found_value
+    local region
+    region="$(get_current_region)"
+    instance_id="$(get_current_instance_id)"
+    found_value="$(get_ec2_tag_value "$region" "$instance_id" "$tag_name")"
+
+    if [ "$tag_value" = "$found_value" ];then
+        return 0
+    fi
+    return 1
+}
+
+trigger_cron() {
+    cd "$MAGENTO_ROOT_DIR"
+    php "$MAGENTO_ROOT_DIR/bin/magento" cron:run
+}
+
+main() {
+    if [ -z "$MAGENTO_ROOT_DIR" ];then
+        echo "MAGENTO_ROOT_DIR env is not set!"
+        exit 1
+    fi
+    while sleep 60;do
+        if is_tag_exists "Cron" "yes";then
+            trigger_cron &
+        fi
+    done
+}
+
+main "$@"