Merge pull request #3 from st-tech/merge-digdag-master

Merge digdag master

Author: sonots

.circleci/config.yml

@@ -2,11 +2,11 @@ version: 2.0
 
 jobs:
   test:
-    parallelism: 4
+    parallelism: 8
 
     # executor type https://circleci.com/docs/2.0/executor-types/
     docker:
-      - image: digdag/digdag-build:20180921T102000-413301ee05f9660e41af70cdcddf0c6e747f0cc1
+      - image: digdag/digdag-build:20191203T225216-c5b7206657c98c728b183634079f0a919ee98f8c
         # environment Variables in a Job https://circleci.com/docs/2.0/env-vars/#setting-an-environment-variable-in-a-job
         environment:
           TERM: dumb
@@ -54,7 +54,7 @@ jobs:
   docs_deployment:
     # executor type https://circleci.com/docs/2.0/executor-types/
     docker:
-      - image: digdag/digdag-build:20180921T191208-befdec51e47abfa34143ee63a8c8929cbfc7c759
+      - image: digdag/digdag-build:20191203T225216-c5b7206657c98c728b183634079f0a919ee98f8c
         # environment Variables in a Job https://circleci.com/docs/2.0/env-vars/#setting-an-environment-variable-in-a-job
         environment:
           TERM: dumb

.gitignore

@@ -6,11 +6,16 @@
 */build/
 */out/
 *.pyc
+venv*
+.venv*
 /dist/*.tar.gz
 /config/test_postgresql.properties
+/config/dummy.properties
 MANIFEST
 pom.xml
 .digdag
+docker/bootstrap/constraints.txt
+docker/bootstrap/requirements.txt
 
 # intellij idea
 *.iml
@@ -20,4 +25,4 @@ pom.xml
 
 # generated self-signed cert
 littleproxy_cert
-littleproxy_keystore.jks
+littleproxy_keystore.jks

.travis.yml

@@ -2,6 +2,8 @@ sudo: required
 
 language: java
 
+dist: trusty
+
 jdk:
   - oraclejdk8
 
@@ -16,8 +18,8 @@ cache:
 env:
   global:
     - TERM=dumb
-    - CI_NODE_TOTAL=8
-    - BUILD_IMAGE=digdag/digdag-build:20180921T191208-befdec51e47abfa34143ee63a8c8929cbfc7c759
+    - CI_NODE_TOTAL=12
+    - BUILD_IMAGE=digdag/digdag-build:20191203T225216-c5b7206657c98c728b183634079f0a919ee98f8c
   matrix:
     - CI_NODE_INDEX=0
     - CI_NODE_INDEX=1
@@ -27,6 +29,10 @@ env:
     - CI_NODE_INDEX=5
     - CI_NODE_INDEX=6
     - CI_NODE_INDEX=7
+    - CI_NODE_INDEX=8
+    - CI_NODE_INDEX=9
+    - CI_NODE_INDEX=10
+    - CI_NODE_INDEX=11
 
 before_install:
   - sudo service --status-all

README.md

@@ -8,21 +8,23 @@
 
 Please check [digdag.io](https://digdag.io) and [docs.digdag.io](https://docs.digdag.io) for installation & user manual.
 
+REST API document is available at [docs.digdag.io/api](http://docs.digdag.io/api/).
+
 ## Development
 
 ### Prerequirements
 
 * JDK 8
-* Node.js 8.x
+* Node.js 12.x
 
 Installing Node.js using nodebrew:
 
 ```
 $ curl -L git.io/nodebrew | perl - setup
 $ echo 'export PATH=$HOME/.nodebrew/current/bin:$PATH' >> ~/.bashrc
 $ source ~/.bashrc
-$ nodebrew install-binary v8.x
-$ nodebrew use v8.x
+$ nodebrew install-binary v12.x
+$ nodebrew use v12.x
 ```
 
 Installing Node.js using Homebrew on Mac OS X:
@@ -36,18 +38,6 @@ $ brew install node
   * sphinx_rtd_theme
   * recommonmark
 
-### Check Digdag REST API
-
-Use `--enable-swagger` option to check the current Digdag REST API.
-
-```
-$ ./gradlew cli
-$ ./pkg/digdag-<current version>.jar server --memory --enable-swagger # Run server with --enable-swagger option
-
-$ docker run -dp 8080:8080 swaggerapi/swagger-ui # Run Swagger-UI on different console
-$ open http://localhost:8080/?url=http://localhost:65432/api/swagger.json # Open api/swagger.json on Swagger-UI
-```
-
 ### Running tests
 
 ```
@@ -86,23 +76,39 @@ It makes an executable in `pkg/`, e.g. `pkg/digdag-$VERSION.jar`.
 ### Releasing a new version
 
 You need to set Bintray user name and API key in `BINTRAY_USER` and `BINTRAY_KEY` environment variables.
-
-1. run `git pull origin --tags`.
-2. run `./gradlew setVersion -Pto=<version>` command.
-3. write release notes to `releases/release-<version>.rst` file. It must include at least version (the first line) and release date (the last line).
-4. run `./gradlew clean cli site check releaseCheck`.
-5. if it succeeded, run `./gradlew release`.
+In the following instructions, assumed that `upstream` is set to `treasure-data/digdag` and `origin` is set to your private repository.
+
+1. run `git pull upstream master --tags`.
+1. run `./gradlew setVersion -Pto=<version>` command.
+1. write release notes to `releases/release-<version>.rst` file. It must include at least version (the first line) and release date (the last line).
+1. run `./gradlew clean cli site check releaseCheck`.
+1. make a release branch. `git checkout -b release_v<version>` and commit.
+1. push the release branch to origin and create a PR.
+1. after the PR is merged to master, checkout master and pull latest upstream/master.
+1. run `./gradlew clean cli site check releaseCheck` again.
+1. if it succeeded, run `./gradlew release`.
+1. a few minutes later, run `digdag selfupdate` and confirm the version.
 
 If major version is incremented, also update `version =` and `release =` at [digdag-docs/src/conf.py](digdag-docs/src/conf.py).
 
+If you are expert, skip 5. to 7. and directly update master branch.
+
+### Post-process of new release
+
+You also need following steps after new version has been released.
+
+1. create a tag `git tag -a v<version>` and push `git push upstream v<version>`
+1. create a release in [GitHub releases](https://github.com/treasure-data/digdag/releases).
+1. create next snapshot version, run `./gradlew setVersion -Pto=<next-version>-SNAPSHOT`.
+1. push to master.
+
 
 ### Releasing a SNAPSHOT version
 
 ```
 ./gradlew releaseSnapshot
 ```
 
-
 ### Develop digdag-ui
 
 Node.js development server is useful because it reloads changes of digdag-ui source code automatically.
@@ -125,17 +131,44 @@ $ npm install
 $ npm run dev    # starts dev server on http://localhost:9000/
 ```
 
+### Updating REST API document
+
+Run this command to update REST API document file at digdag-docs/src/api/swagger.yaml.
+
+```
+./gradlew swaggerYaml  # dump swagger.yaml file
+```
+
+Use `--enable-swagger` option to check the current Digdag REST API.
+
+```
+$ ./gradlew cli
+$ ./pkg/digdag-<current version>.jar server --memory --enable-swagger # Run server with --enable-swagger option
+
+$ docker run -dp 8080:8080 swaggerapi/swagger-ui # Run Swagger-UI on different console
+$ open http://localhost:8080/?url=http://localhost:65432/api/swagger.json # Open api/swagger.json on Swagger-UI
+```
 
 ### Updating documents
 
 Documents are in digdag-docs/src directory. They're built using Sphinx.
 
 Website is hosted on [www.digdag.io](http://www.digdag.io) using Github Pages. Pages are built using deployment step of circle.yml and automatically pushed to [gh-pages branch of digdag-docs repository](https://github.com/treasure-data/digdag-docs/tree/gh-pages).
 
-To build the pages and check them locally, run following command:
+To build the pages and check them locally, follow this instruction.
+
+Create a virtual environment of Python and install dependent Python libraries including Sphinx.
+
+```
+$ python3 -m venv .venv
+$ source .venv/bin/activate
+(.venv)$ pip install -r digdag-docs/requirements.txt -c digdag-docs/constraints.txt
+```
+
+After installation of Python libraries, You can build with running the following command:
 
 ```
-$ ./gradlew site
+(.venv)$ ./gradlew site
 ```
 
 This might not always update all necessary files (Sphinx doesn't manage update dependencies well). In this case, run `./gradlew clean` first.
@@ -153,7 +186,7 @@ The list of release note is [here](https://github.com/treasure-data/digdag/tree/
 Digdag is using a Java annotation processor `org.immutables:value.` The combination of Java annotation processing and Gradle on IntelliJ IDEA sometimes introduces some troubles. In Digdag's case, you may run into some compile errors like `cannot find symbol: class ImmutableRestWorkflowDefinitionCollection.`
 So we'd recommend the followings to avoid those compile errors if you want to develop Digdag one the IDE.
 
-1. There's an important configuration option to be enabled to fully have IntelliJ be fully integrated with an existing gradle build configuration: `Delegate IDE build/run actions to gradle` needs to be enabled. 
+1. There's an important configuration option to be enabled to fully have IntelliJ be fully integrated with an existing gradle build configuration: `Delegate IDE build/run actions to gradle` needs to be enabled.
 
 ![](https://user-images.githubusercontent.com/17990895/48221255-9706be80-e35f-11e8-8283-1ca6d713e31c.png)
 

appveyor.yml

@@ -18,7 +18,7 @@ environment:
   - JAVA_HOME: C:\Program Files (x86)\Java\jdk1.8.0
 
 install:
-  - ps: Install-Product node '8'
+  - ps: Install-Product node '12'
   - node --version
   - npm --version
 

build.gradle

@@ -1,8 +1,8 @@
 plugins {
-    id 'com.github.johnrengelman.shadow' version '4.0.4'
-    id 'com.jfrog.bintray' version '1.8.4'
+    id 'com.github.johnrengelman.shadow' version '2.0.3'
+    id 'com.jfrog.bintray' version '1.8.1'
     id 'com.github.kt3k.coveralls' version '2.6.3'
-    id 'com.github.spotbugs' version '1.6.9'
+    id 'com.github.spotbugs' version '1.6.2'
     id 'net.ltgt.apt-idea' version '0.19'
 }
 
@@ -12,7 +12,7 @@ apply plugin: 'maven'
 
 allprojects {
     group = 'io.digdag'
-    version = '0.9.33'
+    version = '0.9.41-SNAPSHOT'
 
     ext {
         isSnapshotRelease = version.endsWith('-SNAPSHOT')
@@ -110,17 +110,18 @@ subprojects {
     }
 
     ext {
-        jacksonVersion = "2.8.11"
+        jacksonVersion = "2.9.9"
+        jacksonDatabindVersion = "2.9.9"
         awsJavaSdkVersion = "1.11.63"
-        guavaVersion = "19.0"
+        guavaVersion = "27.1-jre"
     }
 
     tasks.withType(JavaCompile) {
         options.compilerArgs << "-Xlint:unchecked" << "-Xlint:deprecation"
     }
 
     spotbugs {
-        toolVersion = '3.1.11'
+        toolVersion = '3.1.5'
         effort = 'max'
         reportLevel = 'high'
         ignoreFailures = true
@@ -137,7 +138,7 @@ subprojects {
     jacocoTestReport {
         // exclude classes generated by Immutables
         afterEvaluate {
-            classDirectories.from = files(classDirectories.files.collect {
+            classDirectories = files(classDirectories.files.collect {
                 fileTree(dir: it, exclude: ['**/Immutable*'])
             })
         }
@@ -286,19 +287,19 @@ subprojects {
 
 jacocoTestReport {
     dependsOn = subprojects.test
-    sourceDirectories.from = files(subprojects.sourceSets.main.allSource.srcDirs)
-    classDirectories.from =  files(subprojects.sourceSets.main.output)
-    executionData.from = files(subprojects.jacocoTestReport.executionData)
+    sourceDirectories = files(subprojects.sourceSets.main.allSource.srcDirs)
+    classDirectories =  files(subprojects.sourceSets.main.output)
+    executionData = files(subprojects.jacocoTestReport.executionData)
     reports {
         xml.enabled = true
         html.enabled = true
     }
     onlyIf = { true }
     doFirst {
-        executionData.from = files(executionData.findAll { it.exists() })
+        executionData = files(executionData.findAll { it.exists() })
     }
     afterEvaluate {
-        classDirectories.from = files(classDirectories.files.collect {
+        classDirectories = files(classDirectories.files.collect {
             fileTree(dir: it, exclude: ['**/Immutable*'])
         })
     }
@@ -311,7 +312,7 @@ coveralls {
 
 wrapper {
     // Find the latest version at https://gradle.org/releases/
-    gradleVersion = '5.2'
+    gradleVersion = '4.8'
 }
 
 project(':digdag-ui') {
@@ -321,7 +322,8 @@ project(':digdag-ui') {
 
         doFirst {
             exec {
-                commandLine System.env.NPM ?: "npm", "install"
+                // npm ci requires npm version 5.7 or more.
+                commandLine System.env.NPM ?: "npm", "ci"
             }
             exec {
                 commandLine System.env.NPM ?: "npm", "run", "build"
@@ -394,7 +396,7 @@ task cli(dependsOn: ':digdag-cli:shadowJar') {
         file('pkg').mkdirs()
         cliBuildFile.write("")
         cliBuildFile.append(file("digdag-cli/src/main/sh/selfrun.sh").readBytes())
-        cliBuildFile.append(file("digdag-cli/build/libs/digdag-cli-${project.version}.jar").readBytes())
+        cliBuildFile.append(file("digdag-cli/build/libs/digdag-cli-${project.version}-all.jar").readBytes())
         cliBuildFile.setExecutable(true)
     }
 }
@@ -406,6 +408,17 @@ task classpath(dependsOn: [':digdag-cli:classpath']){
     }
 }
 
+task swaggerYaml(dependsOn: ':digdag-cli:shadowJar') {
+    doLast {
+        file("config/dummy.properties").write("")
+        Process server = ["java", "-jar", "digdag-cli/build/libs/digdag-cli-${project.version}-all.jar", "server", "-c", "config/dummy.properties", "--memory", "--enable-swagger"].execute()
+        sleep(5 * 1000)
+        ["curl", "http://127.0.0.1:65432/api/swagger.yaml", "-o", "digdag-docs/src/_extra/api/swagger.yaml"].execute().waitFor()
+        server.destroy()
+        server.waitForOrKill(1000)
+    }
+}
+
 clean {
     delete 'classpath'
     delete 'pkg'

ci/run_td_tests.sh

@@ -11,6 +11,16 @@ idleTimeout = 10
 minimumPoolSize = 0
 "
 
+echo "---TARGET test ---"
+target_src=`circleci tests glob "digdag-tests/src/test/java/acceptance/td/**/*IT.java" | circleci tests split --split-by=timings`
+echo $target_src | xargs -n 1 echo
+echo "------------------"
+
+
 export CI_ACCEPTANCE_TEST=true
 
-./gradlew clean cleanTest test --info --no-daemon -p digdag-tests --tests 'acceptance.td.*'
+./gradlew clean cleanTest test --info --no-daemon -p digdag-tests --tests 'acceptance.td.*' \
+	  -PtestFilter="$target_src"
+#	  -PtestFilter='`circleci tests glob "digdag-tests/src/test/java/acceptance/td/**/*.java" | circleci tests split --split-by=timings`'
+
+

ci/run_test_docker.sh

@@ -1,6 +1,7 @@
 #!/bin/bash -xe
 
 docker run \
+-v /var/run/docker.sock:/var/run/docker.sock \
 -w /digdag \
 -v `pwd`/:/digdag \
 -v ~/.gradle:/root/.gradle \

digdag-cli/build.gradle

@@ -14,4 +14,5 @@ dependencies {
     // This line is for including digdag-storage-s3 in CLI (shadow jar) used through
     // dynamic class loading. digdag-cli itself doesn't depend on digdag-storage-s3.
     compile project(':digdag-storage-s3')
+    compile project(':digdag-storage-gcs')
 }