paginate api gateway resources to support more routes

* update route change detection to account for pagination

.gitignore

@@ -1,12 +1,13 @@
 *.gem
 *.rbc
 .bundle
+.byebug_history
 .config
 .yardoc
-InstalledFiles
 _yardoc
 coverage
 doc/
+InstalledFiles
 lib/bundler/man
 pkg
 rdoc
@@ -15,10 +16,9 @@ test/tmp
 test/version_tmp
 tmp
 
-spec/fixtures/project/handlers
 .codebuild/definitions
-demo*
 /html
+/demo
+Gemfile.lock
 spec/fixtures/apps/franky/dynamodb/migrate
-Gemfile.lock 
-.byebug_history
+spec/fixtures/project/handlers

lib/jets/cfn/builders/api_gateway_builder.rb

@@ -12,9 +12,9 @@ def initialize(options={})
     def compose
       return unless @options[:templates] || @options[:stack_type] != :minimal
 
-      add_gateway_rest_api
-      add_custom_domain
-      add_gateway_routes
+      add_gateway_routes # "child template": build before add_gateway_rest_api. RestApi logical id and change detection is dependent on it.
+      add_gateway_rest_api # changes parent template
+      add_custom_domain    # changes parent template
     end
 
     # template_path is an interface method
@@ -57,23 +57,14 @@ def add_route53_dns
     end
 
     # Adds route related Resources and Outputs
+    # Delegates to ApiResourcesBuilder
+    PAGE_LIMIT = Integer(ENV['JETS_AWS_OUTPUTS_LIMIT'] || 60) # Allow override for testing
     def add_gateway_routes
-      # The routes required a Gateway Resource to contain them.
-      # TODO: Support more routes. Right now outputing all routes in 1 template will hit the 60 routes limit.
-      # Will have to either output them as a joined string or break this up to multiple templates.
-      # http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/cloudformation-limits.html
-      # Outputs: Maximum number of outputs that you can declare in your AWS CloudFormation template. 60 outputs
-      # Output name: Maximum size of an output name. 255 characters.
-      #
-      # Note we must use .all_paths, not .routes here because we need to
-      # build the parent ApiGateway::Resource nodes also
-      Jets::Router.all_paths.each do |path|
-        homepage = path == ''
-        next if homepage # handled by RootResourceId output already
-
-        resource = Jets::Resource::ApiGateway::Resource.new(path, internal: true)
-        add_resource(resource)
-        add_outputs(resource.outputs)
+      # Reject homepage. Otherwise we have 60 - 1 resources on the first page.
+      # There's a next call in ApiResources.add_gateway_resources to skip the homepage.
+      all_paths = Jets::Router.all_paths.reject { |p| p == '' }
+      all_paths.each_slice(PAGE_LIMIT).each_with_index do |paths, i|
+        ApiResourcesBuilder.new(@options, paths, i+1).build
       end
     end
   end

lib/jets/cfn/builders/api_resources_builder.rb

@@ -0,0 +1,46 @@
+module Jets::Cfn::Builders
+  class ApiResourcesBuilder
+    include Interface
+    include Jets::AwsServices
+
+    def initialize(options={}, paths=[], page)
+      @options, @paths, @page = options, paths, page
+      @template = ActiveSupport::HashWithIndifferentAccess.new(Resources: {})
+    end
+
+    # compose is an interface method
+    def compose
+      return unless @options[:templates] || @options[:stack_type] != :minimal
+
+      add_rest_api_parameter
+      add_gateway_routes
+    end
+
+    # template_path is an interface method
+    def template_path
+      Jets::Naming.api_resources_template_path(@page)
+    end
+
+    def add_rest_api_parameter
+      add_parameter("RestApi", Description: "RestApi")
+    end
+
+    def add_gateway_routes
+      @paths.each do |path|
+        homepage = path == ''
+        next if homepage # handled by RootResourceId output already
+
+        resource = Jets::Resource::ApiGateway::Resource.new(path)
+        add_resource(resource)
+        add_outputs(resource.outputs)
+
+        parent_path = resource.parent_path_parameter
+        add_parameter(parent_path) unless part_of_template?(parent_path)
+      end
+    end
+
+    def part_of_template?(parent_path)
+      @template["Resources"].key?(parent_path)
+    end
+  end
+end

lib/jets/cfn/builders/parent_builder.rb

@@ -57,6 +57,7 @@ def build_child_resources
 
       if full? and !Jets::Router.routes.empty?
         add_api_gateway
+        add_api_resources
         add_api_deployment
       end
     end
@@ -80,6 +81,20 @@ def add_api_gateway
       add_child_resources(resource)
     end
 
+    def add_api_resources
+      expression = "#{Jets::Naming.template_path_prefix}-api-resources-*"
+      # IE: path: #{Jets.build_root}/templates/demo-dev-2-api-resources-1.yml"
+      Dir.glob(expression).sort.each do |path|
+        next unless File.file?(path)
+
+        regexp = Regexp.new("#{Jets.config.project_namespace}-api-resources-(\\d+).yml") # tricky to escape \d pattern
+        md = path.match(regexp)
+        page = md[1]
+        resource = Jets::Resource::ChildStack::ApiResource.new(@options[:s3_bucket], page: page)
+        add_child_resources(resource)
+      end
+    end
+
     def add_api_deployment
       resource = Jets::Resource::ChildStack::ApiDeployment.new(@options[:s3_bucket])
       add_child_resources(resource)

lib/jets/cfn/built_template.rb

@@ -0,0 +1,15 @@
+module Jets::Cfn
+  # Caches the built template to reduce filesystem IO calls.
+  class BuiltTemplate
+    class << self
+      @@cache = {}
+      def get(path)
+        if @@cache[path]
+          @@cache[path] # using cache
+        else
+          @@cache[path] = YAML.load_file(path) # setting and using cache
+        end
+      end
+    end
+  end
+end

lib/jets/naming.rb

@@ -30,6 +30,10 @@ def api_gateway_template_path
       "#{template_path_prefix}-api-gateway.yml"
     end
 
+    def api_resources_template_path(page)
+      "#{template_path_prefix}-api-resources-#{page}.yml"
+    end
+
     def api_deployment_template_path
       "#{template_path_prefix}-api-deployment.yml"
     end

lib/jets/resource/api_gateway/resource.rb

@@ -37,16 +37,20 @@ def desc
       path.empty? ? 'Homepage route: /' : "Route for: /#{path}"
     end
 
-    def parent_id
+    def parent_path_parameter
       if @path.include?('/') # posts/:id or posts/:id/edit
         parent_path = @path.split('/')[0..-2].join('/')
         parent_logical_id = path_logical_id(parent_path)
-        "!Ref " + Jets::Resource.truncate_id("#{parent_logical_id}ApiResource")
+        Jets::Resource.truncate_id("#{parent_logical_id}ApiResource")
       else
-        "!GetAtt #{RestApi.logical_id(@internal)}.RootResourceId"
+        "RootResourceId"
       end
     end
 
+    def parent_id
+      "!Ref " + parent_path_parameter
+    end
+
     def path_part
       last_part = path.split('/').last
       last_part.split('/').map {|s| transform_capture(s) }.join('/') if last_part

lib/jets/resource/api_gateway/rest_api/change_detection.rb

@@ -4,39 +4,7 @@ class ChangeDetection
     include Jets::AwsServices
 
     def changed?
-      return false unless parent_stack_exists?
-      current_binary_media_types != new_binary_media_types ||
       Routes.changed?
     end
-
-    def new_binary_media_types
-      rest_api = Jets::Resource::ApiGateway::RestApi.new
-      rest_api.binary_media_types
-    end
-    memoize :new_binary_media_types
-
-    # Duplicated in rest_api/change_detection.rb, base_path/role.rb, rest_api/routes.rb
-    def current_binary_media_types
-      return nil unless parent_stack_exists?
-
-      stack = cfn.describe_stacks(stack_name: parent_stack_name).stacks.first
-
-      api_gateway_stack_arn = lookup(stack[:outputs], "ApiGateway")
-
-      stack = cfn.describe_stacks(stack_name: api_gateway_stack_arn).stacks.first
-      rest_api_id = lookup(stack[:outputs], "RestApi")
-
-      resp = apigateway.get_rest_api(rest_api_id: rest_api_id)
-      resp.binary_media_types
-    end
-    memoize :current_binary_media_types
-
-    def parent_stack_exists?
-      stack_exists?(parent_stack_name)
-    end
-
-    def parent_stack_name
-      Jets::Naming.parent_stack_name
-    end
   end
 end

lib/jets/resource/api_gateway/rest_api/routes/change.rb

@@ -1,8 +1,16 @@
 # Detects route changes
 class Jets::Resource::ApiGateway::RestApi::Routes
   class Change
+    include Jets::AwsServices
+
     def changed?
-      To.changed? || Variable.changed? || ENV['JETS_REPLACE_API']
+      return false unless parent_stack_exists?
+
+      MediaTypes.changed? || To.changed? || Variable.changed? || Page.changed? || ENV['JETS_REPLACE_API']
+    end
+
+    def parent_stack_exists?
+      stack_exists?(Jets::Naming.parent_stack_name)
     end
   end
 end

lib/jets/resource/api_gateway/rest_api/routes/change/base.rb

@@ -3,6 +3,10 @@ class Base
     extend Memoist
     include Jets::AwsServices
 
+    def self.changed?
+      new.changed?
+    end
+
     # Build up deployed routes from the existing CloudFormation resources.
     def deployed_routes
       routes = []

lib/jets/resource/api_gateway/rest_api/routes/change/media_types.rb

@@ -0,0 +1,36 @@
+class Jets::Resource::ApiGateway::RestApi::Routes::Change
+  class MediaTypes < Base
+    def changed?
+      current_binary_media_types != new_binary_media_types
+    end
+
+    def new_binary_media_types
+      rest_api = Jets::Resource::ApiGateway::RestApi.new
+      rest_api.binary_media_types
+    end
+    memoize :new_binary_media_types
+
+    def current_binary_media_types
+      return nil unless parent_stack_exists?
+
+      stack = cfn.describe_stacks(stack_name: parent_stack_name).stacks.first
+
+      api_gateway_stack_arn = lookup(stack[:outputs], "ApiGateway")
+
+      stack = cfn.describe_stacks(stack_name: api_gateway_stack_arn).stacks.first
+      rest_api_id = lookup(stack[:outputs], "RestApi")
+
+      resp = apigateway.get_rest_api(rest_api_id: rest_api_id)
+      resp.binary_media_types
+    end
+    memoize :current_binary_media_types
+
+    def parent_stack_exists?
+      stack_exists?(parent_stack_name)
+    end
+
+    def parent_stack_name
+      Jets::Naming.parent_stack_name
+    end
+  end
+end

lib/jets/resource/api_gateway/rest_api/routes/change/page.rb

@@ -0,0 +1,93 @@
+class Jets::Resource::ApiGateway::RestApi::Routes::Change
+  class Page < Base
+    def changed?
+      route_page_moved? || old_api_template?
+    end
+
+    def route_page_moved?
+      moved?(new_pages, deployed_pages)
+    end
+
+    # Routes page to logical ids
+    def moved?(new_pages, deployed_pages)
+      not_moved = true # page has not moved
+      new_pages.each do |logical_id, new_page_number|
+        if !deployed_pages[logical_id].nil? && deployed_pages[logical_id] != new_page_number
+          not_moved = false # page has moved
+          break
+        end
+      end
+      !not_moved # moved
+    end
+
+    def new_pages
+      local_logical_ids_map
+    end
+    memoize :new_pages
+
+    def deployed_pages
+      remote_logical_ids_map
+    end
+    memoize :deployed_pages
+
+    # logical id to page map
+    # Important: In Cfn::Builders::ApiGatewayBuilder, the add_gateway_routes and ApiResourcesBuilder needs to run
+    # before the parent add_gateway_rest_api method.
+    def local_logical_ids_map(path_expression="#{Jets::Naming.template_path_prefix}-api-resources-*.yml")
+      logical_ids = {} # logical id => page number
+
+      Dir.glob(path_expression).each do |path|
+        md = path.match(/-api-resources-(\d+).yml/)
+        page_number = md[1]
+
+        template = Jets::Cfn::BuiltTemplate.get(path)
+        template['Resources'].keys.each do |logical_id|
+          logical_ids[logical_id] = page_number
+        end
+      end
+
+      logical_ids
+    end
+
+    # aws cloudformation describe-stack-resources --stack-name demo-dev-ApiResources1-DYGLIEY3VAWT | jq -r '.StackResources[].LogicalResourceId'
+    def remote_logical_ids_map
+      logical_ids = {} # logical id => page number
+
+      parent_resources.each do |resource|
+        stack_name = resource.physical_resource_id # full physical id can be used as stack name also
+        regexp = Regexp.new("#{Jets.config.project_namespace}-ApiResources(\\d+)-") # tricky to escape \d pattern
+        md = stack_name.match(regexp)
+        if md
+          page_number = md[1]
+
+          resp = cfn.describe_stack_resources(stack_name: stack_name)
+          resp.stack_resources.map(&:logical_resource_id).each do |logical_id|
+            logical_ids[logical_id] = page_number
+          end
+        end
+      end
+
+      logical_ids
+    end
+
+    def old_api_template?
+      logical_resource_ids = parent_resources.map(&:logical_resource_id)
+
+      api_gateway_found = logical_resource_ids.detect do |logical_id|
+        logical_id == "ApiGateway"
+      end
+      return false unless api_gateway_found
+
+      api_resources_found = logical_resource_ids.detect do |logical_id|
+        logical_id.match(/^ApiResources\d+$/)
+      end
+      !api_resources_found # if api_resources_found then it's the new structure. so opposite is old structure
+    end
+
+    def parent_resources
+      resp = cfn.describe_stack_resources(stack_name: Jets::Naming.parent_stack_name)
+      resp.stack_resources
+    end
+    memoize :parent_resources
+  end
+end