Update main with changes in dev branch (#2291)

.circleci/config.yml

@@ -49,7 +49,7 @@ jobs:
       - run: # run a command
           name: Generate config
           command: |
-            python .circleci/scripts/config-generator.py -d ${CIRCLE_TAG:-none} -o .circleci/continuation-config.yml
+            python .circleci/scripts/config-generator.py -d ${CIRCLE_TAG:-none} -o .circleci/continuation-config.yml -e ${CIRCLE_PR_REPONAME:-none}
       - continuation/continue:
           configuration_path: .circleci/continuation-config.yml # use newly generated config to continue
 workflows:

.circleci/scripts/config-generator.py

@@ -7,13 +7,15 @@
 
 
 def runCommand(argv: List[str]):
-    deploy = False
-    output_filepath = ".circleci/continuation-config.yml"
+
+    deploy: bool = False
+    external_build: bool = False
+    output_filepath: str = ".circleci/continuation-config.yml"
     arg_help = "{0} -d <deploy?> -o <output>".format(argv[0])
 
     print(argv)
     try:
-        opts, _ = getopt.getopt(argv[1:], "hd:o:")
+        opts, _ = getopt.getopt(argv[1:], "hd:o:e:")
     except:
         print(arg_help)
         sys.exit(2)
@@ -33,8 +35,16 @@ def runCommand(argv: List[str]):
             print("deploy arg -- " + str(arg) + " -- " + str(deploy))
         elif opt in ("-o", "--output"):
             output_filepath = arg
-
-    createConfigFile(deploy, output_filepath)
+        elif opt in ("-e", "--external"):
+            external_build = arg is not None and arg not in [
+                "none",
+                "None",
+                "False",
+                "false",
+                "f",
+            ]
+            print("Building for external PR " + str(external_build))
+    createConfigFile(deploy, output_filepath, external_build)
 
 
 def setup():
@@ -58,12 +68,7 @@ def setup():
         common_jobs = yaml.safe_load(cf)
 
 
-def getTagRegexString(connector_names: List[str]):
-    # version_regex = "([0-9]+)\\.([0-9]+)\\.([0-9]+)(?:-\\w+)?"
-    # tags = connector_names + ["all"]
-    # tagExpression = "(" + "|".join(tags) + ")"
-    # return f"/^{version_regex}\\/{tagExpression}$/"
-
+def getTagRegexString(connector_names: List[str]) -> str:
     # Version format 'x.y.z' with optional suffix '-{SUFFIX_NAME}' and optional '/all' ending to force build all tags
     return "/^([0-9]+)\\.([0-9]+)\\.([0-9]+)(?:-\\w+)?(\\/all)?$/"
 
@@ -75,10 +80,10 @@ def getTagFilter(connector_names: List[str]):
     }
 
 
-def createConfigFile(deploy: bool, outputPath: str):
+def createConfigFile(deploy: bool, outputPath: str, external_build: bool):
     print("---- Started creating config ----")
     print(
-        f"\n  -- Settings --\n  Deploy: {deploy}\n  Output path: {outputPath}\n  --\n"
+        f"\n  -- Settings --\n  Deploy: {deploy}\n  Output path: {outputPath}\n  External build: {external_build}\n  --\n"
     )
     setup()
 
@@ -169,18 +174,52 @@ def createConfigFile(deploy: bool, outputPath: str):
                 jobAttrs["filters"] = getTagFilter(slugs_to_match)
                 print(f"Added missing filter to job: {x[0]}")
 
+        jobsToWait = []
         for jobName in jobs_before_deploy:
-            main_workflow["jobs"] += [getNewDeployJob(jobName)]
+            job = getNewDeployJob(jobName)
+            if job["deploy-connector-new"]:
+                jobsToWait.append(job["deploy-connector-new"]["name"])
+            main_workflow["jobs"] += [job]
+        main_workflow["jobs"] += [
+            {
+                "notify-deploy": {
+                    "requires": jobsToWait,
+                    "context": "discord",
+                    "filters": getTagFilter(slugs_to_match),
+                }
+            }
+        ]
+    if external_build:
+        removeStepsThatUseSecrets(config)
     # Output continuation file
     with open(outputPath, "w") as file:
         yaml.dump(config, file, sort_keys=False)
 
     print("---- Finished creating config ----")
 
 
+def removeStepsThatUseSecrets(config):
+    jobs: dict[str, dict[str, dict]] = config["workflows"]["build"]["jobs"]
+    filteredJobs = []
+
+    for jobItem in jobs:
+        key = next(iter(jobItem.keys()))
+        if key == "get-ci-tools":
+            continue
+        jobDict: dict = jobItem[key]
+        requires = jobDict["requires"]
+        if requires:
+            if "get-ci-tools" in requires:
+                requires.pop(requires.index("get-ci-tools"))
+        filteredJobs.append({f"{key}": jobDict})
+
+    config["workflows"]["build"]["jobs"] = filteredJobs
+    print("Cleaned up config for external build")
+
+
 def getNewDeployJob(jobName: str):
-    slug = jobName.split("-build")[0]
-    isMac = jobName.find("-mac") != -1
+    slug: str = jobName.split("-build")[0]
+    isMac: bool = jobName.find("-mac") != -1
     deployJob: Dict[str, Any] = {
         "slug": slug.split("-mac")[0] if isMac else slug,
         "name": slug + "-deploy-mac" if isMac else slug + "-deploy",
@@ -189,7 +228,7 @@ def getNewDeployJob(jobName: str):
         "extension": "zip" if isMac else "exe",
         "requires": ["deploy-connectors", jobName],
         "filters": getTagFilter([jobName]),
-        "context": "do-spaces-speckle-releases",
+        "context": ["do-spaces-speckle-releases"],
     }
     return {"deploy-connector-new": deployJob}
 

.circleci/scripts/config-template.yml

@@ -5,6 +5,7 @@ orbs:
   aws-s3: circleci/[email protected]
   codecov: codecov/[email protected]
   wait-for: cobli/[email protected]
+  discord: antonioned/[email protected]
 
 commands:
   packandpublish:
@@ -21,6 +22,18 @@ commands:
             dotnet nuget push "**/*.nupkg" -s https://api.nuget.org/v3/index.json -k $env:NUGET_APIKEY -n --skip-duplicate
           environment:
             WORKFLOW_NUM: << pipeline.number >>
+  cached-checkout:
+    steps:
+      - restore_cache:
+          keys:
+            - &source-cache source-v1-{{ .Branch }}-{{ .Revision }}
+            - source-v1-{{ .Branch }}-
+            - source-v1-
+      - checkout
+      - save_cache:
+          key: *source-cache
+          paths:
+            - ".git"
   packandpublish-bash:
     parameters:
       projectfilepath:
@@ -38,11 +51,10 @@ commands:
             WORKFLOW_NUM: << pipeline.number >>
 jobs: # Each project will have individual jobs for each specific task it has to execute (build, release...)
   build-core:
-    executor: # Using a win executor since there are post-steps in the nuget workflow that use powershell
-      name: win/default
-      shell: powershell.exe
+    docker:
+      - image: mcr.microsoft.com/dotnet/sdk:6.0
     steps:
-      - checkout
+      - cached-checkout
       - run:
           name: Build Core
           command: dotnet build Core/Core.sln -c Release -v q
@@ -79,61 +91,49 @@ jobs: # Each project will have individual jobs for each specific task it has to
           S3_CREATE_BUCKET: "true"
           WAIT_HOSTS: 127.0.0.1:5432, 127.0.0.1:6379, 127.0.0.1:9000
     steps:
-      - checkout
+      - cached-checkout
       - run:
           name: Unit Test
           command: dotnet test Core/Tests/TestsUnit.csproj -c Release -v q --logger:"junit;LogFileName={assembly}.results.xml" --results-directory=TestResults --collect:"XPlat Code Coverage" -- DataCollectionRunSettings.DataCollectors.DataCollector.Configuration.Format=opencover
       - run:
           name: Integration Tests # The integration tests are runinng on our test sevrer
           command: dotnet test Core/IntegrationTests/TestsIntegration.csproj -c Release -v q --logger:"junit;LogFileName={assembly}.results.xml" --results-directory=TestResults --collect:"XPlat Code Coverage" -- DataCollectionRunSettings.DataCollectors.DataCollector.Configuration.Format=opencover
-      - store_test_results:
-          path: TestResults
       - store_artifacts:
           path: TestResults
 
       #- codecov/upload
 
   build-objects:
-    macos:
-      xcode: 12.5.1
+    docker:
+      - image: mcr.microsoft.com/dotnet/sdk:6.0
     steps:
-      - checkout
-      - run:
-          name: Install mono
-          command: |
-            HOMEBREW_NO_AUTO_UPDATE=1 HOMEBREW_NO_INSTALL_CLEANUP=1 brew install mono mono-libgdiplus
-      - run:
-          name: Install dotnet
-          command: |
-            curl -sSL https://dot.net/v1/dotnet-install.sh | bash /dev/stdin
-            curl -sSL https://dot.net/v1/dotnet-install.sh | bash /dev/stdin --channel 3.1
+      - cached-checkout
       - run:
           name: Restore Objects
-          command: ~/.dotnet/dotnet restore Objects/Objects.sln
+          command: | # Restore only the projects we're about to build.
+            dotnet restore Objects/Objects/Objects.csproj
+            dotnet restore Objects/Tests/Tests.csproj
       - run:
           name: Build Objects
-          command: ~/.dotnet/dotnet build --no-restore Objects/Objects/Objects.csproj -c Release /p:WarningLevel=0 /p:IsDesktopBuild=false
+          command: dotnet build --no-restore Objects/Objects/Objects.csproj -c Release /p:WarningLevel=0 /p:IsDesktopBuild=false
       - run:
           name: Test Objects
-          command: ~/.dotnet/dotnet test Objects/Tests/Tests.csproj --no-restore -c Release -v q --logger:"junit;LogFileName={assembly}.results.xml" --results-directory=TestResults --collect:"XPlat Code Coverage" -- DataCollectionRunSettings.DataCollectors.DataCollector.Configuration.Format=opencover
+          command: dotnet test Objects/Tests/Tests.csproj --no-restore -c
+            Release -v q --logger:"junit;LogFileName={assembly}.results.xml" --results-directory=TestResults
+            --collect:"XPlat Code Coverage" -- DataCollectionRunSettings.DataCollectors.DataCollector.Configuration.Format=opencover
       - store_test_results:
           path: TestResults
       - store_artifacts:
           path: TestResults
-      # Upload to codecov if necessary
-      # Currently disabled until we figure out a strategy to
-      # upload Core and Objects at once
-      # - codecov/upload
 
   build-desktopui:
-    executor: # Using a win executor since there are post-steps in the nuget workflow that use powershell
-      name: win/default
-      shell: powershell.exe
+    docker:
+      - image: mcr.microsoft.com/dotnet/sdk:6.0
     steps:
-      - checkout
+      - cached-checkout
       - run:
           name: Build DesktopUI2
-          command: dotnet build DesktopUI2/DesktopUI2.sln -c Release -v q
+          command: dotnet build DesktopUI2/DesktopUI2/DesktopUI2.csproj -c Release -v q
 
   build-connector: # Reusable job for basic connectors
     executor:
@@ -144,7 +144,7 @@ jobs: # Each project will have individual jobs for each specific task it has to
         type: string
       projname:
         type: string
-        default: ''
+        default: ""
       dllname:
         type: string
       slug:
@@ -157,7 +157,7 @@ jobs: # Each project will have individual jobs for each specific task it has to
         type: boolean
         default: false
     steps:
-      - checkout
+      - cached-checkout
       - attach_workspace:
           at: ./
       - run:
@@ -166,16 +166,16 @@ jobs: # Each project will have individual jobs for each specific task it has to
       - when:
           condition: << parameters.build-with-msbuild >>
           steps:
-          - run:
-              name: Build << parameters.slnname >>
-              command: |
-                $tag = if([string]::IsNullOrEmpty($env:CIRCLE_TAG)) { "2.0.999" } else { $env:CIRCLE_TAG }
-                $semver = if($tag.Contains('/')) {$tag.Split("/")[0] } else { $tag }
-                $ver = if($semver.Contains('-')) {$semver.Split("-")[0] } else { $semver }
-                $version = "$($ver).$($env:WORKFLOW_NUM)"
-                msbuild << parameters.slnname >>/<< parameters.slnname >>.sln /p:Configuration=Release /p:WarningLevel=0 /p:IsDesktopBuild=false /p:AssemblyVersionNumber=$version /p:AssemblyInformationalVersion=$semver /p:Version=$semver
-              environment:
-                WORKFLOW_NUM: << pipeline.number >>
+            - run:
+                name: Build << parameters.slnname >>
+                command: |
+                  $tag = if([string]::IsNullOrEmpty($env:CIRCLE_TAG)) { "2.0.999" } else { $env:CIRCLE_TAG }
+                  $semver = if($tag.Contains('/')) {$tag.Split("/")[0] } else { $tag }
+                  $ver = if($semver.Contains('-')) {$semver.Split("-")[0] } else { $semver }
+                  $version = "$($ver).$($env:WORKFLOW_NUM)"
+                  msbuild << parameters.slnname >>/<< parameters.slnname >>.sln /p:Configuration=Release /p:WarningLevel=0 /p:IsDesktopBuild=false /p:AssemblyVersionNumber=$version /p:AssemblyInformationalVersion=$semver /p:Version=$semver
+                environment:
+                  WORKFLOW_NUM: << pipeline.number >>
       - unless:
           condition: << parameters.build-with-msbuild >>
           steps:
@@ -188,7 +188,11 @@ jobs: # Each project will have individual jobs for each specific task it has to
                   $version = "$($ver).$($env:WORKFLOW_NUM)"
                   dotnet publish << parameters.slnname >>/<< parameters.slnname >>/<< parameters.projname >>.csproj -c Release -v q -r win-x64 --self-contained /p:WarningLevel=0 /p:AssemblyVersionNumber=$version /p:AssemblyInformationalVersion=$semver /p:Version=$semver
                 environment:
-                  WORKFLOW_NUM: << pipeline.number >> 
+                  WORKFLOW_NUM: << pipeline.number >>
+      - run:
+          name: Exit if External PR
+          shell: bash.exe
+          command: if [ "$CIRCLE_PR_REPONAME" ]; then circleci-agent step halt; fi
       - run:
           name: Create Innosetup signing cert
           command: |
@@ -231,7 +235,6 @@ jobs: # Each project will have individual jobs for each specific task it has to
         type: string
         default: Any
     steps:
-      - checkout
       - attach_workspace:
           at: ./
       - run:
@@ -277,7 +280,7 @@ jobs: # Each project will have individual jobs for each specific task it has to
         type: string
         default: ""
     steps:
-      - checkout
+      - cached-checkout
       - attach_workspace:
           at: ./
       - run:
@@ -366,6 +369,9 @@ jobs: # Each project will have individual jobs for each specific task it has to
                   mkdir -p speckle-sharp-ci-tools/Mac/<< parameters.installername >>/.installationFiles/
                   cp << parameters.slnname >>/<< parameters.slnname >>/bin/Release/net6.0/osx-x64/publish/<< parameters.slug >>-mac.zip speckle-sharp-ci-tools/Mac/<<parameters.installername>>/.installationFiles
       # Create installer
+      - run:
+          name: Exit if External PR
+          command: if [ "$CIRCLE_PR_REPONAME" ]; then circleci-agent step halt; fi
       - run:
           name: Build Mac installer
           command: ~/.dotnet/dotnet publish speckle-sharp-ci-tools/Mac/<<parameters.installername>>/<<parameters.installername>>.sln -r osx-x64 -c Release
@@ -424,7 +430,7 @@ jobs: # Each project will have individual jobs for each specific task it has to
       name: << parameters.e >>
       shell: bash.exe
     steps:
-      - checkout
+      - cached-checkout
       - attach_workspace:
           at: ./
       - run:
@@ -444,7 +450,7 @@ jobs: # Each project will have individual jobs for each specific task it has to
             cd ConnectorArchicad/AddOn
             mkdir Build.Win.x64.<<parameters.archicadversion>>
             export PATH=$PATH:"C:\Program Files\CMake\bin"
-            cmake -B "./Build.Win.x64.<<parameters.archicadversion>>/" -A "x64" -T "v142" -DAC_API_DEVKIT_DIR="../../Resources/Archicad<<parameters.archicadversion>>DevKit" -DAC_MDID_DEV=$GRAPHISOFT_DEV_ID -DAC_MDID_LOC=$GRAPHISOFT_ADDON_ID
+            cmake -B "./Build.Win.x64.<<parameters.archicadversion>>/" -A "x64" -T "v142" -DAC_API_DEVKIT_DIR="../../Resources/Archicad<<parameters.archicadversion>>DevKit" -DAC_MDID_DEV=${GRAPHISOFT_DEV_ID:-1} -DAC_MDID_LOC=${GRAPHISOFT_ADDON_ID:-1}
       - run:
           name: Build add-on
           command: |
@@ -469,7 +475,7 @@ jobs: # Each project will have individual jobs for each specific task it has to
         type: string
         default: ""
     steps:
-      - checkout
+      - cached-checkout
       - attach_workspace:
           at: ./
       - run:
@@ -488,7 +494,7 @@ jobs: # Each project will have individual jobs for each specific task it has to
           command: |
             cd ConnectorArchicad/AddOn
             mkdir Build.macOS.x64.<<parameters.archicadversion>>
-            cmake -B "./Build.macOS.x64.<<parameters.archicadversion>>/" -G 'Xcode' -DCMAKE_OSX_ARCHITECTURES=x86_64 -DAC_API_DEVKIT_DIR="../../Resources/Archicad<<parameters.archicadversion>>DevKitMac" -DAC_MDID_DEV=$GRAPHISOFT_DEV_ID -DAC_MDID_LOC=$GRAPHISOFT_ADDON_ID
+            cmake -B "./Build.macOS.x64.<<parameters.archicadversion>>/" -G 'Xcode' -DCMAKE_OSX_ARCHITECTURES=x86_64 -DAC_API_DEVKIT_DIR="../../Resources/Archicad<<parameters.archicadversion>>DevKitMac" -DAC_MDID_DEV=${GRAPHISOFT_DEV_ID:-1} -DAC_MDID_LOC=${GRAPHISOFT_ADDON_ID:-1}
       - run:
           name: Build add-on
           command: |
@@ -518,7 +524,16 @@ jobs: # Each project will have individual jobs for each specific task it has to
       - run:
           name: Proceed to deploy
           command: echo "This step is just here to wait for all build jobs before proceeding to deploy each of them individually. If a job fails, no connector will be deployed."
-
+  notify-deploy:
+    docker:
+      - image: cimg/base:2021.01
+    steps:
+      - discord/status:
+          mentions: "1067457311980933140"
+          success_message:
+            ":tada: a new version of Speckle-Sharp connectors was build
+            successfully!"
+          failure_message: ":red_circle: oh no! Speckle-Sharp connectors build has failed!"
 # The main workflows for our monorepo pipeline.
 # There should be at least one workflow per project in the monorepo. Each workflow should be run only when a boolean parameter is passed that corresponds to the pattern 'run_{PROJECT_NAME}'.
 # These parameters are set by the 'selective-ci' job.