serverless.yml

service: stock-prices-etl

provider:
  name: google
  runtime: nodejs10
  region: us-east1
  project: bigquery-stock-av
  # The GCF credentials can be a little tricky to set up. Luckily we've documented this for you here:
  # https://serverless.com/framework/docs/providers/google/guide/credentials/
  #
  # the path to the credentials file needs to be absolute
  credentials: ~/.gcloud/keyfile.json
  environment:
    RAW_STOCK_DATA_STORAGE: ${self:custom.rawStockDataStorage}
    PARSED_STOCK_DATA_STORAGE: ${self:custom.parsedStockDataStorage}
    STOCK_PIPELINE_QUEUE_NAME: ${self:custom.stockPipelineQueueName}
    STOCK_BIGQUERY_DATASET_NAME: ${self:custom.stockBigqueryDatasetName}
    STOCK_BIGQUERY_TABLE_NAME: ${self:custom.stockBigqueryTableName}
    BASE_URL: 'https://www.alphavantage.co/query'
    SIMULTANEOUS_STOCK_DOWNLOADS: '4'
    DEDUPLICATED_STOCK_BIGQUERY_TABLE_NAME: ${self:custom.deduplicatedStockBigqueryTableName}
    STOCK_METADATA_BIGQUERY_DATASET_NAME: ${self:custom.stockMetadataBigqueryDatasetName}
    JOB_BIGQUERY_TABLE_NAME: ${self:custom.jobBigqueryTableName}
    ALL_STOCKS_BIGQUERY_TABLE_NAME: ${self:custom.allStocksBigqueryTableName}

plugins:
  - serverless-google-cloudfunctions

package:
  exclude:
    - node_modules/**
    - .gitignore
    - .git/**

custom:
  stockPipelineQueueName: stockPipelineQueue
  stockSelectorTriggerName: stockSelectorTrigger
  rawStockDataStorage: raw-stock-data-storage-1062880978334
  parsedStockDataStorage: parsed-stock-data-storage-1062880978334
  stockBigqueryDatasetName: stock_dataset
  stockBigqueryTableName: stock_table
  dailyJobsTriggerQueueName: dailyJobsTriggerQueue
  deduplicationJobTriggerQueueName: deduplicationJobTriggerQueue
  dailySummariesDatamartName: daily_summaries_datamart
  deduplicatedStockBigqueryTableName: deduplicated_stock_table
  stockMetadataBigqueryDatasetName: stock_metadata_dataset
  jobBigqueryTableName: job_table
  allStocksBigqueryTableName: all_stocks_table

functions:
  stockSelector:
    handler: stockSelector
    events:
      - event:
          eventType: providers/cloud.pubsub/eventTypes/topic.publish
          resource: projects/${self:provider.project}/topics/${self:custom.stockSelectorTriggerName}

  extractStockData:
    handler: extractStockData
    events:
      - event:
          eventType: providers/cloud.pubsub/eventTypes/topic.publish
          resource: projects/${self:provider.project}/topics/${self:custom.stockPipelineQueueName}

  transformStockData:
    handler: transformStockData
    events:
      - event:
          eventType: google.storage.object.finalize
          resource: projects/${self:provider.project}/buckets/${self:custom.rawStockDataStorage}

  loadStockData:
    handler: loadStockData
    events:
      - event:
          eventType: google.storage.object.finalize
          resource: projects/${self:provider.project}/buckets/${self:custom.parsedStockDataStorage}

  dailyJobsTrigger:
    handler: dailyJobsTrigger
    events:
      - event:
          eventType: providers/cloud.pubsub/eventTypes/topic.publish
          resource: projects/${self:provider.project}/topics/${self:custom.dailyJobsTriggerQueueName}

  deduplicationJobTrigger:
    handler: deduplicationJobTrigger
    events:
      - event:
          eventType: providers/cloud.pubsub/eventTypes/topic.publish
          resource: projects/${self:provider.project}/topics/${self:custom.deduplicationJobTriggerQueueName}

resources:
  resources:
    # GCS
    - name: ${self:custom.rawStockDataStorage}
      type: storage.v1.bucket
      properties:
        project: ${self:provider.project}
        name: ${self:custom.rawStockDataStorage}
        lifecycle:
          rule:
          - action:
              type: Delete
            condition:
              age: 5
    - name: ${self:custom.parsedStockDataStorage}
      type: storage.v1.bucket
      properties:
        project: ${self:provider.project}
        name: ${self:custom.parsedStockDataStorage}

    # PubSub
    - name: ${self:custom.stockPipelineQueueName}
      type: pubsub.v1.topic
      properties:
        topic: ${self:custom.stockPipelineQueueName}
    - name: ${self:custom.stockSelectorTriggerName}
      type: pubsub.v1.topic
      properties:
        topic: ${self:custom.stockSelectorTriggerName}
    - name: ${self:custom.dailyJobsTriggerQueueName}
      type: pubsub.v1.topic
      properties:
        topic: ${self:custom.dailyJobsTriggerQueueName}
    - name: ${self:custom.deduplicationJobTriggerQueueName}
      type: pubsub.v1.topic
      properties:
        topic: ${self:custom.deduplicationJobTriggerQueueName}
    
    # Bigquery
    - name: ${self:custom.dailySummariesDatamartName}
      type: bigquery.v2.dataset
      properties:
        name: ${self:custom.dailySummariesDatamartName}
        datasetReference:
          datasetId: ${self:custom.dailySummariesDatamartName}
    - name: ${self:custom.stockBigqueryDatasetName}
      type: bigquery.v2.dataset
      properties:
        name: ${self:custom.stockBigqueryDatasetName}
        datasetReference:
          datasetId: ${self:custom.stockBigqueryDatasetName}
    - name: ${self:custom.stockBigqueryTableName}
      type: bigquery.v2.table
      properties:
        tableReference:
          tableId: ${self:custom.stockBigqueryTableName}
        name: ${self:custom.stockBigqueryTableName}
        datasetId: $(ref.${self:custom.stockBigqueryDatasetName}.datasetReference.datasetId)
        schema:
          fields:
            - name: date
              type: DATE
            - name: symbol
              type: STRING
            - name: open
              type: NUMERIC
            - name: close
              type: NUMERIC
            - name: low
              type: NUMERIC
            - name: high
              type: NUMERIC
            - name: volume
              type: NUMERIC
            - name: extracted_at
              type: TIMESTAMP
            - name: transformed_at
              type: TIMESTAMP
            - name: tick
              type: STRING
        timePartitioning:
          type: DAY
          field: date
        clustering:
          fields: 
            - symbol
    - name: ${self:custom.stockMetadataBigqueryDatasetName}
      type: bigquery.v2.dataset
      properties:
        name: ${self:custom.stockMetadataBigqueryDatasetName}
        datasetReference:
          datasetId: ${self:custom.stockMetadataBigqueryDatasetName}
    - name: ${self:custom.jobBigqueryTableName}
      type: bigquery.v2.table
      properties:
        tableReference:
          tableId: ${self:custom.jobBigqueryTableName}
        name: ${self:custom.jobBigqueryTableName}
        datasetId: $(ref.${self:custom.stockMetadataBigqueryDatasetName}.datasetReference.datasetId)
        schema:
          fields:
            - name: name
              type: STRING
            - name: destination_dataset
              type: STRING
            - name: destination_table
              type: STRING
            - name: query
              type: STRING
    - name: ${self:custom.allStocksBigqueryTableName}
      type: bigquery.v2.table
      properties:
        tableReference:
          tableId: ${self:custom.allStocksBigqueryTableName}
        name: ${self:custom.allStocksBigqueryTableName}
        datasetId: $(ref.${self:custom.stockMetadataBigqueryDatasetName}.datasetReference.datasetId)
        schema:
          fields:
            - name: symbol
              type: STRING