edx · nadeemshahzad · Mar 19, 2025 · Mar 21, 2025 · Mar 26, 2025 · Mar 26, 2025
diff --git a/Makefile b/Makefile
@@ -20,6 +20,11 @@ requirements:
 	pip install -qr requirements/pip.txt --exists-action w
 	pip install -qr requirements.txt --exists-action w
 
+requirements3_12:
+	pip install -qr requirements/pip.txt --exists-action w
+	pip install -qr requirements3_12txt --exists-action w
+
+
 COMMON_CONSTRAINTS_TXT=requirements/common_constraints.txt
 .PHONY: $(COMMON_CONSTRAINTS_TXT)
 $(COMMON_CONSTRAINTS_TXT):

diff --git a/playbooks/create_rds.yml b/playbooks/create_rds.yml
@@ -83,5 +83,5 @@
     changed_when: "add_role_result.rc == 0"
     when: cluster_name is defined and cluster_role_arn is defined
 
-- include: create_db_and_users.yml
+- import_tasks: create_db_and_users.yml
   when: database_connection.login_host is defined
diff --git a/playbooks/edx_provision.yml b/playbooks/edx_provision.yml
@@ -61,7 +61,7 @@
       wait_for:
         path: /var/log/cloud-init.log
         timeout: 15
-        search_regex: "final-message"
+        search_regex: "final[-_]message"
     - name: gather_facts
       setup: ""
   vars_files:

diff --git a/playbooks/library/ec2_lookup b/playbooks/library/ec2_lookup
@@ -16,7 +16,10 @@
 
 from __future__ import absolute_import
 from __future__ import print_function
-import six
+import sys
+import boto3
+from ansible.module_utils.basic import AnsibleModule
+
 DOCUMENTATION = '''
 ---
 module: ec2_lookup
@@ -28,8 +31,8 @@ options:
   region:
     description:
       - The AWS region to use.  Must be specified if ec2_url
-        is not used.  If not specified then the value of the
-        EC2_REGION environment variable, if any, is used.
+        is not used.  If not specified then the value of
+        the EC2_REGION environment variable, if any, is used.
     required: false
     default: null
     aliases: [ 'aws_region', 'ec2_region' ]
@@ -42,20 +45,20 @@ options:
     aliases: [ 'ec2_secret_key', 'secret_key' ]
   aws_access_key:
     description:
-      - AWS access key. If not set then the value of the
-        AWS_ACCESS_KEY environment variable is used.
+      - AWS access key. If not set then the value of
+        the AWS_ACCESS_KEY environment variable is used.
     required: false
     default: null
     aliases: [ 'ec2_access_key', 'access_key' ]
   tags:
-    desription:
+    description:
       - tags to lookup
     required: false
     default: null
     type: dict
     aliases: []
 
-requirements: [ "boto" ]
+requirements: [ "boto3" ]
 author: John Jarvis
 '''
 
@@ -70,8 +73,6 @@ EXAMPLES = '''
         Name: foo
 '''
 
-import sys
-
 AWS_REGIONS = ['ap-northeast-1',
                'ap-southeast-1',
                'ap-southeast-2',
@@ -81,17 +82,8 @@ AWS_REGIONS = ['ap-northeast-1',
                'us-west-1',
                'us-west-2']
 
-try:
-    import boto.ec2
-    from boto.ec2 import connect_to_region
-except ImportError:
-    print("failed=True msg='boto required for this module'")
-    sys.exit(1)
-
-
 def main():
-
-    module=AnsibleModule(
+    module = AnsibleModule(
         argument_spec=dict(
             ec2_url=dict(),
             region=dict(aliases=['aws_region', 'ec2_region'],
@@ -108,35 +100,34 @@ def main():
     region = module.params.get('region')
     ec2_url = module.params.get('ec2_url')
 
-    # If we have a region specified, connect to its endpoint.
+    # If region is specified, create a session with boto3 and connect to the EC2 service
     if region:
         try:
-            ec2 = connect_to_region(region, aws_access_key_id=aws_access_key,
-                                    aws_secret_access_key=aws_secret_key)
-        except boto.exception.NoAuthHandlerFound as e:
-            module.fail_json(msg=str(e))
-    # If we specified an ec2_url then try connecting to it
-    elif ec2_url:
-        try:
-            ec2 = boto.connect_ec2_endpoint(ec2_url, aws_access_key,
-                                            aws_secret_key)
-        except boto.exception.NoAuthHandlerFound as e:
-            module.fail_json(msg=str(e))
+            ec2 = boto3.resource('ec2', region_name=region, aws_access_key_id=aws_access_key, aws_secret_access_key=aws_secret_key)
+        except Exception as e:
+            module.fail_json(msg=f"Error connecting to AWS EC2: {str(e)}")
     else:
-        module.fail_json(msg="Either region or ec2_url must be specified")
+        module.fail_json(msg="Region must be specified")
+
+    # If EC2 URL is provided, it can be used for connection
+    if ec2_url:
+        module.fail_json(msg="ec2_url is not supported in boto3 connection. Please use region instead.")
 
     instances = []
     instance_ids = []
-    for res in ec2.get_all_instances(filters={'tag:' + tag: value
-                                     for tag, value in six.iteritems(module.params.get('tags'))}):
-        for inst in res.instances:
-            if inst.state == "running":
-                instances.append({k: v for k, v in six.iteritems(inst.__dict__)
-                                  if isinstance(v, (six.string_types))})
-                instance_ids.append(inst.id)
-    module.exit_json(changed=False, instances=instances,
-                     instance_ids=instance_ids)
 
+    # Get all instances with the specified tags
+    filters = {'tag:' + tag: value for tag, value in module.params.get('tags', {}).items()}
+    try:
+        instances_query = ec2.instances.filter(Filters=[{'Name': f'tag:{tag}', 'Values': [value]} for tag, value in filters.items()])
+        for instance in instances_query:
+            if instance.state['Name'] == 'running':
+                instances.append({k: v for k, v in instance.__dict__.items() if isinstance(v, str)})
+                instance_ids.append(instance.id)
+    except Exception as e:
+        module.fail_json(msg=f"Error querying EC2 instances: {str(e)}")
+
+    module.exit_json(changed=False, instances=instances, instance_ids=instance_ids)
 
 # this is magic, see lib/ansible/module_common.py
 #<<INCLUDE_ANSIBLE_MODULE_COMMON>>

diff --git a/playbooks/roles/aws/defaults/main.yml b/playbooks/roles/aws/defaults/main.yml
@@ -36,15 +36,25 @@ aws_region: "us-east-1"
 aws_s3cmd: "/usr/bin/s3cmd"
 aws_cmd: "/usr/local/bin/aws"
 aws_requirements: "{{ vhost_dirs.home.path }}/requirements.txt"
+aws_venv_dir: "{{ vhost_dirs.home.path }}/venv"
 
 #
 # OS packages
 #
 
-aws_debian_pkgs:
-  - python-setuptools
+aws_debian_pkgs: "{{ aws_debian_pkgs_default + aws_release_specific_debian_pkgs[ansible_distribution_release] }}"
+
+aws_debian_pkgs_default:
   - s3cmd
 
+aws_release_specific_debian_pkgs:
+  bionic:
+    - python-setuptools
+  focal:
+    - python-setuptools
+  noble:
+    - python3-setuptools
+
 aws_redhat_pkgs: []
 
 # The AWS_GATHER_FACTS switch is used to enable/disable data gathering

diff --git a/playbooks/roles/aws/tasks/main.yml b/playbooks/roles/aws/tasks/main.yml
@@ -71,6 +71,15 @@
     requirements: "{{ aws_requirements }}"
     state: present
     extra_args: "-i {{ COMMON_PYPI_MIRROR_URL }}"
+  when: ansible_distribution_release != 'noble'
+
+- name: Install aws python packages
+  pip:
+    requirements: "{{ aws_requirements }}"
+    virtualenv: "{{ aws_venv_dir }}"
+    state: present
+    extra_args: "-i {{ COMMON_PYPI_MIRROR_URL }}"
+  when: ansible_distribution_release == 'noble'
 
 - name: Copy the boto global config file
   template:

diff --git a/playbooks/roles/common/tasks/main.yml b/playbooks/roles/common/tasks/main.yml
@@ -2,7 +2,9 @@
 - name: Check Configuration Sources
   fail:
     msg: "Configuration Sources Checking (COMMON_EXTRA_CONFIGURATION_SOURCES_CHECKING) is enabled, you must define {{ item }}"
-  when: COMMON_EXTRA_CONFIGURATION_SOURCES_CHECKING and ({{ item }} is not defined or {{ item }} != True)
+  when: 
+    - COMMON_EXTRA_CONFIGURATION_SOURCES_CHECKING
+    - item not in vars or vars[item] != True
   with_items: "{{ COMMON_EXTRA_CONFIGURATION_SOURCES }}"
   tags:
     - "install"
@@ -88,7 +90,7 @@
   until: add_repo is success
   retries: 10
   delay: 5
-  when: ansible_distribution_release == 'bionic' or ansible_distribution_release == 'focal' or ansible_distribution_release == 'jammy'
+  when: ansible_distribution_release == 'noble' or ansible_distribution_release == 'bionic' or ansible_distribution_release == 'focal' or ansible_distribution_release == 'jammy'
   tags:
     - install
     - install:system-requirements
@@ -170,8 +172,15 @@
     state: present
     extra_args: "-i {{ COMMON_PYPI_MIRROR_URL }}"
     executable: pip3
-  when: ansible_distribution in common_debian_variants
+  when: ansible_distribution in common_debian_variants and ansible_distribution_release != "noble"
 
+- name: pip install virtualenv
+  pip:
+    name: "{{ common_pip_pkgs }}"
+    state: present
+    extra_args: "-i {{ COMMON_PYPI_MIRROR_URL }} --break-system-packages"
+    executable: pip3
+  when: ansible_distribution in common_debian_variants and ansible_distribution_release == "noble"
 
 - name: update /etc/hosts
   template:

diff --git a/playbooks/roles/common_vars/defaults/main.yml b/playbooks/roles/common_vars/defaults/main.yml
@@ -149,6 +149,8 @@ common_release_specific_debian_pkgs:
     - python3.5-dev
   jammy:
     - python3.8
+  noble: 
+    - python3.12
 
 common_debian_pkgs: "{{ common_debian_pkgs_default + common_release_specific_debian_pkgs[ansible_distribution_release] }}"
 
@@ -160,7 +162,12 @@ old_python_debian_pkgs:
   - "python2.7=2.7.10-0+{{ ansible_distribution_release }}1"
 
 
-COMMON_PIP_VERSION: '21.2.1'
+COMMON_PIP_VERSION: "{{ common_release_specific_pip_version[ansible_distribution_release] }}"
+
+common_release_specific_pip_version:
+  bionic: "21.2.1"
+  focal: "21.2.1"
+  noble: "24.0"
 
 common_pip_pkgs:
   - pip=={{ COMMON_PIP_VERSION }}

diff --git a/playbooks/roles/config-encoders/README.rst b/playbooks/roles/config-encoders/README.rst
@@ -1014,10 +1014,10 @@ data structure is the following::
                 - ^name: reload
                 - allow:
                   - ^user: root
-        - include:
+        - import_tasks:
           - ^ignore_missing: "yes"
           - /etc/oddjobd.conf.d/*.conf
-        - include:
+        - import_tasks:
           - ^ignore_missing: "yes"
           - /etc/oddjobd-local.conf
 

diff --git a/playbooks/roles/demo/tasks/main.yml b/playbooks/roles/demo/tasks/main.yml
@@ -37,4 +37,6 @@
     owner: "{{ demo_edxapp_user }}"
     group: "{{ common_web_group }}"
 
-- include: deploy.yml tags=deploy
+- import_tasks: deploy.yml 
+  tags:
+   - deploy
diff --git a/playbooks/roles/ecommerce/defaults/main.yml b/playbooks/roles/ecommerce/defaults/main.yml
@@ -334,11 +334,10 @@ ecommerce_debian_pkgs:
   - python3-dev
 
 ecommerce_release_specific_debian_pkgs:
-  xenial:
-    - python-dev
   bionic:
     - python-dev
   focal: []
+  noble: []
 
 ecommerce_redhat_pkgs: []
 

diff --git a/playbooks/roles/edx_ansible/defaults/main.yml b/playbooks/roles/edx_ansible/defaults/main.yml
@@ -23,28 +23,35 @@ edx_ansible_debian_running_services:
   - fail2ban
 
 edx_ansible_debian_pkgs_default:
-  - python-apt
   - libmysqlclient-dev
   - git-core
   - build-essential
   - libxml2-dev
   - libxslt1-dev
   - curl
-  - python-yaml
   - python3-pip
   - python3-mysqldb
 
 edx_ansible_release_specific_debian_pkgs:
   xenial:
+    - python-apt
     - python-pip
     - python-mysqldb
     - python-dev
   bionic:
+    - python-apt
     - python-pip
     - python-mysqldb
     - python-dev
   focal:
+    - python-yaml
+    - python-apt
     - python3-dev
+  noble:
+    - python3-apt
+    - python3-yaml
+    - python3-virtualenv
+    - python3.12-dev
 
 edx_ansible_debian_pkgs: "{{ edx_ansible_debian_running_services + edx_ansible_debian_pkgs_default + edx_ansible_release_specific_debian_pkgs[ansible_distribution_release] }}"
 

diff --git a/playbooks/roles/edx_ansible/tasks/main.yml b/playbooks/roles/edx_ansible/tasks/main.yml
@@ -53,7 +53,7 @@
   tags:
     - install:system-requirements
 
-- include: deploy.yml
+- import_tasks: deploy.yml
   tags:
     - deploy
 

diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml
@@ -1807,14 +1807,13 @@ edxapp_debian_pkgs:
   - libsqlite3-dev
 
 edxapp_release_specific_debian_pkgs:
-  xenial:
-    - ipython
-    - python-dev
   bionic:
     - ipython
     - python-dev
   focal:
     - ipython3
+  noble:
+    - ipython3
 
 edxapp_debian_pkgs_default: "{{ edxapp_debian_pkgs + edxapp_release_specific_debian_pkgs[ansible_distribution_release] }}"
 

diff --git a/playbooks/roles/edxapp/tasks/deploy.yml b/playbooks/roles/edxapp/tasks/deploy.yml
@@ -454,7 +454,7 @@
 # creates the supervisor jobs for the
 # service variants configured, runs
 # gather_assets and db migrations
-- include: service_variant_config.yml
+- import_tasks: service_variant_config.yml
   tags:
     - service_variant_config
     - deploy
@@ -519,9 +519,10 @@
     - install:configuration
     - install:code
 
-- include: tag_ec2.yml tags=deploy
+- import_tasks: tag_ec2.yml
   when: COMMON_TAG_EC2_INSTANCE
   tags:
+    - deploy
     - remove
     - aws
 

diff --git a/playbooks/roles/edxapp/tasks/main.yml b/playbooks/roles/edxapp/tasks/main.yml
@@ -205,12 +205,12 @@
     - install:base
 
 # Set up the python sandbox execution environment
-- include: python_sandbox_env.yml
+- import_tasks: python_sandbox_env.yml
   when: EDXAPP_PYTHON_SANDBOX
   tags:
     - deploy
 
-- include: deploy.yml
+- import_tasks: deploy.yml
   tags:
     - deploy
-Original file line number
+Diff line change
@@ Expand Up / @@ -53,7 +53,7 @@ @@
       tags:
         - install:system-requirements
-    - include: deploy.yml
+    - import_tasks: deploy.yml
       tags:
         - deploy
@@ Expand Down @@