Ensure s3 bucket issue 495 - Bucket Tags

docs/config.rst

@@ -90,6 +90,32 @@ See the `CloudFormation Limits Reference`_.
 
 .. _`CloudFormation Limits Reference`: http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/cloudformation-limits.html
 
+
+S3 Bucket tags
+----
+
+Various resources in AWS support arbitrary key-value pair tags. You can set
+the `bucket_tags` Top Level Keyword to populate tags on all S3 buckets Staker
+attempts to create for CloudFormation template uploads, inclduing the S3 bucket
+created by the aws_lambda pre-hook.
+
+If bucket_tags is not set in your Configuration, stacker will fallback to the
+method used to determine tags in your config by the `tags` top level keyword.
+The `bucket_tags` keyword takes precedence over `tags` when applying. Example::
+
+  bucket_tags:
+    "hello": world
+    "my_tag:with_colons_in_key": ${dynamic_tag_value_from_my_env}
+    simple_tag: simple value
+
+If you prefer to have no tags applied to your stacks (versus the default tags
+that stacker applies), specify an empty map for the top-level keyword::
+
+  bucket_tags: {}
+
+S3 Bucket Tags updates get applied on every stacker run
+
+
 Module Paths
 ------------
 When setting the ``classpath`` for blueprints/hooks, it is sometimes desirable to
@@ -236,23 +262,32 @@ the build action::
 Tags
 ----
 
-CloudFormation supports arbitrary key-value pair tags. All stack-level, including automatically created tags, are
-propagated to resources that AWS CloudFormation supports. See `AWS CloudFormation Resource Tags Type`_ for more details.
-If no tags are specified, the `stacker_namespace` tag is applied to your stack with the value of `namespace` as the
-tag value.
+Various resources in AWS support arbitrary key-value pair tags. You can set
+the `tags` Top Level Keyword to populate tags on all Resources that Staker
+attempts to create via CloudFormation. All CloudFormation stack-level resources,
+including automatically created tags, are propagated to resources that AWS
+CloudFormation supports. See `AWS CloudFormation Resource Tags Type`_ for
+more details.
+
+If no tags are specified, the `stacker_namespace` tag is applied to your stack
+with the value of `namespace` as the tag value.
 
-If you prefer to apply a custom set of tags, specify the top-level keyword `tags` as a map. Example::
+If you prefer to apply a custom set of tags, specify the top-level keyword
+`tags` as a map. Example::
 
   tags:
     "hello": world
     "my_tag:with_colons_in_key": ${dynamic_tag_value_from_my_env}
     simple_tag: simple value
 
+
 If you prefer to have no tags applied to your stacks (versus the default tags that stacker applies), specify an empty
 map for the top-level keyword::
 
   tags: {}
 
+Tags updates get applied on every stacker run
+
 .. _`AWS CloudFormation Resource Tags Type`: http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-resource-tags.html
 
 Mappings

stacker/actions/base.py

@@ -86,7 +86,8 @@ def bucket_region(self):
 
     def ensure_cfn_bucket(self):
         """The CloudFormation bucket where templates will be stored."""
-        ensure_s3_bucket(self.s3_conn, self.bucket_name, self.bucket_region)
+        ensure_s3_bucket(self.s3_conn, self.bucket_name, self.bucket_region,
+                         self.context)
 
     def stack_template_url(self, blueprint):
         return stack_template_url(

stacker/config/__init__.py

@@ -351,6 +351,8 @@ class Config(Model):
 
     tags = DictType(StringType, serialize_when_none=False)
 
+    bucket_tags = DictType(StringType, serialize_when_none=False)
+
     mappings = DictType(
         DictType(DictType(StringType)), serialize_when_none=False)
 

stacker/context.py

@@ -102,6 +102,19 @@ def tags(self):
             return {"stacker_namespace": self.namespace}
         return {}
 
+    @property
+    def s3_bucket_tags(self):
+        s3_bucket_tags = self.config.bucket_tags
+        if s3_bucket_tags is not None:
+            return s3_bucket_tags
+        else:
+            s3_bucket_tags = self.config.tags
+            if s3_bucket_tags is not None:
+                return s3_bucket_tags
+            if self.namespace:
+                return {"stacker_namespace": self.namespace}
+            return {}
+
     @property
     def _base_fqn(self):
         return self.namespace.replace(".", "-").lower()

stacker/hooks/aws_lambda.py

@@ -490,7 +490,7 @@ def create_template(self):
     session = get_session(bucket_region)
     s3_client = session.client('s3')
 
-    ensure_s3_bucket(s3_client, bucket_name, bucket_region)
+    ensure_s3_bucket(s3_client, bucket_name, bucket_region, context)
 
     prefix = kwargs.get('prefix', '')
 

stacker/tests/actions/test_base.py

@@ -42,6 +42,17 @@ def test_ensure_cfn_bucket_exists(self):
                 "Bucket": ANY,
             }
         )
+        stubber.add_response(
+            "put_bucket_tagging",
+            service_response={},
+            expected_params={
+                "Bucket": ANY,
+                "Tagging": {
+                    "TagSet": [
+                        {"Key": "stacker_namespace",
+                         "Value": u"mynamespace"}]}
+            }
+        )
         with stubber:
             action.ensure_cfn_bucket()
 
@@ -65,6 +76,17 @@ def test_ensure_cfn_bucket_doesnt_exist_us_east(self):
                 "Bucket": ANY,
             }
         )
+        stubber.add_response(
+            "put_bucket_tagging",
+            service_response={},
+            expected_params={
+                "Bucket": ANY,
+                "Tagging": {
+                    "TagSet": [
+                        {"Key": "stacker_namespace",
+                         "Value": u"mynamespace"}]}
+            }
+        )
         with stubber:
             action.ensure_cfn_bucket()
 
@@ -91,6 +113,17 @@ def test_ensure_cfn_bucket_doesnt_exist_us_west(self):
                 }
             }
         )
+        stubber.add_response(
+            "put_bucket_tagging",
+            service_response={},
+            expected_params={
+                "Bucket": ANY,
+                "Tagging": {
+                    "TagSet": [
+                        {"Key": "stacker_namespace",
+                         "Value": u"mynamespace"}]}
+            }
+        )
         with stubber:
             action.ensure_cfn_bucket()
 

stacker/util.py

@@ -16,10 +16,10 @@
 from collections import OrderedDict
 
 import botocore.client
+from git import Repo
 import botocore.exceptions
 import dateutil
 import yaml
-from git import Repo
 from yaml.constructor import ConstructorError
 from yaml.nodes import MappingNode
 
@@ -503,7 +503,7 @@ def s3_bucket_location_constraint(region):
     return region
 
 
-def ensure_s3_bucket(s3_client, bucket_name, bucket_region):
+def ensure_s3_bucket(s3_client, bucket_name, bucket_region, context):
     """Ensure an s3 bucket exists, if it does not then create it.
 
     Args:
@@ -512,9 +512,19 @@ def ensure_s3_bucket(s3_client, bucket_name, bucket_region):
         bucket_name (str): The bucket being checked/created.
         bucket_region (str, optional): The region to create the bucket in. If
             not provided, will be determined by s3_client's region.
+        context (:class:`stacker.context.Context`): The stacker context, used
+            set the S3 bucket tags from the stacker config
+
     """
     try:
+        # Checking is bucket exists
         s3_client.head_bucket(Bucket=bucket_name)
+        # pulling tags from s3_bucket_tags function
+        tagset = _s3_bucket_tags(context)
+        # setting tags on every run - must have permission to perform
+        # the s3:PutBucketTagging action
+        s3_client.put_bucket_tagging(Bucket=bucket_name,
+                                     Tagging={'TagSet': tagset})
     except botocore.exceptions.ClientError as e:
         if e.response['Error']['Message'] == "Not Found":
             logger.debug("Creating bucket %s.", bucket_name)
@@ -526,7 +536,13 @@ def ensure_s3_bucket(s3_client, bucket_name, bucket_region):
                 create_args["CreateBucketConfiguration"] = {
                     "LocationConstraint": location_constraint
                 }
+            # pulling tags from s3_bucket_tags function
+            tagset = _s3_bucket_tags(context)
             s3_client.create_bucket(**create_args)
+            # setting tags on every run - must have permission to perform
+            # the s3:PutBucketTagging action
+            s3_client.put_bucket_tagging(Bucket=bucket_name,
+                                         Tagging={'TagSet': tagset})
         elif e.response['Error']['Message'] == "Forbidden":
             logger.exception("Access denied for bucket %s.  Did " +
                              "you remember to use a globally unique name?",
@@ -538,6 +554,20 @@ def ensure_s3_bucket(s3_client, bucket_name, bucket_region):
             raise
 
 
+def _s3_bucket_tags(context):
+    """Returns the tags to be applied for a S3 bucket.
+
+    Args:
+        context (:class:`stacker.context.Context`): The stacker context, used
+            set the S3 bucket tags from the stacker config
+
+    Returns:
+        List of dictionaries containing tags to apply to that bucket.
+    """
+    return [
+        {'Key': t[0], 'Value': t[1]} for t in context.s3_bucket_tags.items()]
+
+
 class Extractor(object):
     """Base class for extractors."""
 
@@ -607,10 +637,9 @@ def extension():
         return '.zip'
 
 
-class SourceProcessor(object):
-    """Makes remote python package sources available in current environment."""
-
-    ISO8601_FORMAT = '%Y%m%dT%H%M%SZ'
+class SourceProcessor():
+    """Makes remote python package sources available in the running python
+       environment."""
 
     def __init__(self, sources, stacker_cache_dir=None):
         """