From e197d205b95cf2ecd44563a39c19010079c4ebaa Mon Sep 17 00:00:00 2001
From: Zhengfei Wang <wangzhengfei.chn@outlook.com>
Date: Tue, 9 Jul 2024 05:49:21 +0000
Subject: [PATCH 01/25] chore: Add license compliance workflow

---
 .github/workflows/license-compliance.yml | 27 ++++++++++++++++++++++++
 1 file changed, 27 insertions(+)
 create mode 100644 .github/workflows/license-compliance.yml

diff --git a/.github/workflows/license-compliance.yml b/.github/workflows/license-compliance.yml
new file mode 100644
index 00000000000..384559e049f
--- /dev/null
+++ b/.github/workflows/license-compliance.yml
@@ -0,0 +1,27 @@
+name: "License Compliance"
+on:
+- pull_request_target
+
+jobs:
+  promptflow-tracing:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event.pull_request.head.sha || github.ref }}
+        fetch-depth: 0
+    - name: merge main to current branch
+      uses: "./.github/actions/step_merge_main"
+    - uses: snok/install-poetry@v1
+    - name: install dependencies
+      run: |
+        poetry install
+        poetry run pip show promptflow-tracing
+        poetry run pip list
+      working-directory: ${{ github.workspace }}/src/promptflow-tracing
+    - name: install pip-licenses
+      run: pip install pip-licenses
+    - name: check licenses
+      run: |
+        pip-licenses --from=mixed --format=csv --output-file=LICENSES.csv
+        cat LICENSES.csv

From 1493595274c073416c5c0e1a1b08945a4b980ae4 Mon Sep 17 00:00:00 2001
From: Zhengfei Wang <wangzhengfei.chn@outlook.com>
Date: Tue, 9 Jul 2024 05:52:02 +0000
Subject: [PATCH 02/25] use pull_request

---
 .github/workflows/license-compliance.yml | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/license-compliance.yml b/.github/workflows/license-compliance.yml
index 384559e049f..4838e44a4f2 100644
--- a/.github/workflows/license-compliance.yml
+++ b/.github/workflows/license-compliance.yml
@@ -1,17 +1,15 @@
 name: "License Compliance"
 on:
-- pull_request_target
+- pull_request
 
 jobs:
   promptflow-tracing:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
       with:
-        ref: ${{ github.event.pull_request.head.sha || github.ref }}
-        fetch-depth: 0
-    - name: merge main to current branch
-      uses: "./.github/actions/step_merge_main"
+        python-version: '3.11'
     - uses: snok/install-poetry@v1
     - name: install dependencies
       run: |

From d774b472d83d107bf460e063c5bedc173ddad1fb Mon Sep 17 00:00:00 2001
From: Zhengfei Wang <wangzhengfei.chn@outlook.com>
Date: Tue, 9 Jul 2024 05:54:14 +0000
Subject: [PATCH 03/25] add pf-core job

---
 .github/workflows/license-compliance.yml | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/license-compliance.yml b/.github/workflows/license-compliance.yml
index 4838e44a4f2..e1b140e9751 100644
--- a/.github/workflows/license-compliance.yml
+++ b/.github/workflows/license-compliance.yml
@@ -14,7 +14,6 @@ jobs:
     - name: install dependencies
       run: |
         poetry install
-        poetry run pip show promptflow-tracing
         poetry run pip list
       working-directory: ${{ github.workspace }}/src/promptflow-tracing
     - name: install pip-licenses
@@ -23,3 +22,23 @@ jobs:
       run: |
         pip-licenses --from=mixed --format=csv --output-file=LICENSES.csv
         cat LICENSES.csv
+
+  promptflow-core:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.11'
+    - uses: snok/install-poetry@v1
+    - name: install dependencies
+      run: |
+        poetry install
+        poetry run pip list
+      working-directory: ${{ github.workspace }}/src/promptflow-core
+    - name: install pip-licenses
+      run: pip install pip-licenses
+    - name: check licenses
+      run: |
+        pip-licenses --from=mixed --format=csv --output-file=LICENSES.csv
+        cat LICENSES.csv

From 81b55381e98ecc746d8e38c660b5cb238aeb7d4c Mon Sep 17 00:00:00 2001
From: Zhengfei Wang <wangzhengfei.chn@outlook.com>
Date: Tue, 9 Jul 2024 06:00:04 +0000
Subject: [PATCH 04/25] use poetry run

---
 .github/workflows/license-compliance.yml | 22 +---------------------
 1 file changed, 1 insertion(+), 21 deletions(-)

diff --git a/.github/workflows/license-compliance.yml b/.github/workflows/license-compliance.yml
index e1b140e9751..6365a060b01 100644
--- a/.github/workflows/license-compliance.yml
+++ b/.github/workflows/license-compliance.yml
@@ -3,26 +3,6 @@ on:
 - pull_request
 
 jobs:
-  promptflow-tracing:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v4
-    - uses: actions/setup-python@v5
-      with:
-        python-version: '3.11'
-    - uses: snok/install-poetry@v1
-    - name: install dependencies
-      run: |
-        poetry install
-        poetry run pip list
-      working-directory: ${{ github.workspace }}/src/promptflow-tracing
-    - name: install pip-licenses
-      run: pip install pip-licenses
-    - name: check licenses
-      run: |
-        pip-licenses --from=mixed --format=csv --output-file=LICENSES.csv
-        cat LICENSES.csv
-
   promptflow-core:
     runs-on: ubuntu-latest
     steps:
@@ -40,5 +20,5 @@ jobs:
       run: pip install pip-licenses
     - name: check licenses
       run: |
-        pip-licenses --from=mixed --format=csv --output-file=LICENSES.csv
+        poetry run pip-licenses --from=mixed --format=csv --output-file=LICENSES.csv
         cat LICENSES.csv

From 5f17ca08951ed0566add77c41eb7ce104196fc9f Mon Sep 17 00:00:00 2001
From: Zhengfei Wang <wangzhengfei.chn@outlook.com>
Date: Tue, 9 Jul 2024 06:11:44 +0000
Subject: [PATCH 05/25] add script to check licenses

---
 .../check-licenses-compliance.py              | 39 +++++++++++++++++++
 1 file changed, 39 insertions(+)
 create mode 100644 scripts/licenses-compliance/check-licenses-compliance.py

diff --git a/scripts/licenses-compliance/check-licenses-compliance.py b/scripts/licenses-compliance/check-licenses-compliance.py
new file mode 100644
index 00000000000..2b198314c89
--- /dev/null
+++ b/scripts/licenses-compliance/check-licenses-compliance.py
@@ -0,0 +1,39 @@
+import csv
+import sys
+import typing
+from pathlib import Path
+
+# list of licenses compatible with MIT
+allowed_licenses = [
+    "MIT",
+    "Apache-2.0",
+    "BSD",
+    "ISC",
+]
+
+
+def check_license_compliance(
+    licenses_file: str,
+    allowed_licenses: typing.List[str],
+) -> typing.List[str]:
+    incompliance_items = list()
+    with open(licenses_file, mode="r") as file:
+        csv_reader = csv.DictReader(file)
+        for row in csv_reader:
+            if row["License"] not in allowed_licenses:
+                incompliance_items.append((row["Name"], row["License"]))
+        return incompliance_items
+
+
+def main(licenses_file: Path):
+    incompliance_items = check_license_compliance(licenses_file, allowed_licenses)
+    if len(incompliance_items) > 0:
+        print("found dependencies with licenses incompliance with MIT license:")
+        for name, license in incompliance_items:
+            print(f"- {name}: {license}")
+    else:
+        print("all dependencies are compliance with MIT license.")
+
+
+if __name__ == "__main__":
+    main(licenses_file=Path(sys.argv[1]).resolve().absolute())

From dc943fdb60eb7a8b8d472e0c9d699d3efa613e81 Mon Sep 17 00:00:00 2001
From: Zhengfei Wang <wangzhengfei.chn@outlook.com>
Date: Tue, 9 Jul 2024 08:25:53 +0000
Subject: [PATCH 06/25] raise exception when incompliance

---
 .../check-licenses-compliance.py              | 23 ++++++++++++++-----
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/scripts/licenses-compliance/check-licenses-compliance.py b/scripts/licenses-compliance/check-licenses-compliance.py
index 2b198314c89..d893e8a02c3 100644
--- a/scripts/licenses-compliance/check-licenses-compliance.py
+++ b/scripts/licenses-compliance/check-licenses-compliance.py
@@ -5,10 +5,15 @@
 
 # list of licenses compatible with MIT
 allowed_licenses = [
-    "MIT",
-    "Apache-2.0",
-    "BSD",
-    "ISC",
+    "Apache Software License",
+    "BSD License",
+    "GNU General Public License (GPL)",
+    "ISC License (ISCL)",
+    "Public Domain",
+    "Python Software Foundation License",
+    "Mozilla Public License 2.0 (MPL 2.0)",
+    "MIT License",
+    "The Unlicense (Unlicense)",
 ]
 
 
@@ -20,8 +25,13 @@ def check_license_compliance(
     with open(licenses_file, mode="r") as file:
         csv_reader = csv.DictReader(file)
         for row in csv_reader:
-            if row["License"] not in allowed_licenses:
-                incompliance_items.append((row["Name"], row["License"]))
+            name = row["Name"]
+            # note that "License" can be a comma separated list of licenses
+            licenses = row["License"]
+            for license in licenses.split(";"):
+                license = license.strip()
+                if license not in allowed_licenses:
+                    incompliance_items.append((name, licenses))
         return incompliance_items
 
 
@@ -31,6 +41,7 @@ def main(licenses_file: Path):
         print("found dependencies with licenses incompliance with MIT license:")
         for name, license in incompliance_items:
             print(f"- {name}: {license}")
+        raise Exception("found dependencies with licenses incompliance with MIT license.")
     else:
         print("all dependencies are compliance with MIT license.")
 

From be9801abb1c376a196cff9042cf8ce9ab3c39e07 Mon Sep 17 00:00:00 2001
From: Zhengfei Wang <wangzhengfei.chn@outlook.com>
Date: Tue, 9 Jul 2024 08:26:08 +0000
Subject: [PATCH 07/25] comment MPL for test

---
 scripts/licenses-compliance/check-licenses-compliance.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/licenses-compliance/check-licenses-compliance.py b/scripts/licenses-compliance/check-licenses-compliance.py
index d893e8a02c3..70a788a6183 100644
--- a/scripts/licenses-compliance/check-licenses-compliance.py
+++ b/scripts/licenses-compliance/check-licenses-compliance.py
@@ -11,7 +11,7 @@
     "ISC License (ISCL)",
     "Public Domain",
     "Python Software Foundation License",
-    "Mozilla Public License 2.0 (MPL 2.0)",
+    # "Mozilla Public License 2.0 (MPL 2.0)",
     "MIT License",
     "The Unlicense (Unlicense)",
 ]

From 909d97bb9cdea587bfabef053b0999ac1e3eda2f Mon Sep 17 00:00:00 2001
From: Zhengfei Wang <wangzhengfei.chn@outlook.com>
Date: Tue, 9 Jul 2024 08:28:16 +0000
Subject: [PATCH 08/25] run script

---
 .github/workflows/license-compliance.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/license-compliance.yml b/.github/workflows/license-compliance.yml
index 6365a060b01..88c99e2a854 100644
--- a/.github/workflows/license-compliance.yml
+++ b/.github/workflows/license-compliance.yml
@@ -21,4 +21,5 @@ jobs:
     - name: check licenses
       run: |
         poetry run pip-licenses --from=mixed --format=csv --output-file=LICENSES.csv
-        cat LICENSES.csv
+        poetry run python ./scripts/licenses-compliance/check-licenses-compliance.py LICENSES.csv
+      working-directory: ${{ github.workspace }}

From 278fad74a87ce2161db4a1240b74f634afc407f1 Mon Sep 17 00:00:00 2001
From: Zhengfei Wang <wangzhengfei.chn@outlook.com>
Date: Tue, 9 Jul 2024 08:31:40 +0000
Subject: [PATCH 09/25] no poetry

---
 .github/workflows/license-compliance.yml | 12 +++---------
 1 file changed, 3 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/license-compliance.yml b/.github/workflows/license-compliance.yml
index 88c99e2a854..595c299845b 100644
--- a/.github/workflows/license-compliance.yml
+++ b/.github/workflows/license-compliance.yml
@@ -10,16 +10,10 @@ jobs:
     - uses: actions/setup-python@v5
       with:
         python-version: '3.11'
-    - uses: snok/install-poetry@v1
-    - name: install dependencies
-      run: |
-        poetry install
-        poetry run pip list
-      working-directory: ${{ github.workspace }}/src/promptflow-core
     - name: install pip-licenses
       run: pip install pip-licenses
     - name: check licenses
       run: |
-        poetry run pip-licenses --from=mixed --format=csv --output-file=LICENSES.csv
-        poetry run python ./scripts/licenses-compliance/check-licenses-compliance.py LICENSES.csv
-      working-directory: ${{ github.workspace }}
+        pip-licenses --from=mixed --format=csv --output-file=LICENSES.csv
+        python ../../scripts/licenses-compliance/check-licenses-compliance.py LICENSES.csv
+      working-directory: ${{ github.workspace }}/src/promptflow-core

From 17f7f2d9ee71313c9ac389245b05e7b236cfdb3c Mon Sep 17 00:00:00 2001
From: Zhengfei Wang <wangzhengfei.chn@outlook.com>
Date: Tue, 9 Jul 2024 08:33:18 +0000
Subject: [PATCH 10/25] pip install & cat

---
 .github/workflows/license-compliance.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/license-compliance.yml b/.github/workflows/license-compliance.yml
index 595c299845b..7b1eece93e2 100644
--- a/.github/workflows/license-compliance.yml
+++ b/.github/workflows/license-compliance.yml
@@ -14,6 +14,8 @@ jobs:
       run: pip install pip-licenses
     - name: check licenses
       run: |
+        pip install .
         pip-licenses --from=mixed --format=csv --output-file=LICENSES.csv
+        cat LICENSES.csv
         python ../../scripts/licenses-compliance/check-licenses-compliance.py LICENSES.csv
       working-directory: ${{ github.workspace }}/src/promptflow-core

From 9b993c117dc2afb45bbecff069319076c058f032 Mon Sep 17 00:00:00 2001
From: Zhengfei Wang <wangzhengfei.chn@outlook.com>
Date: Tue, 9 Jul 2024 08:37:09 +0000
Subject: [PATCH 11/25] refine

---
 scripts/licenses-compliance/check-licenses-compliance.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/scripts/licenses-compliance/check-licenses-compliance.py b/scripts/licenses-compliance/check-licenses-compliance.py
index 70a788a6183..1354c591ded 100644
--- a/scripts/licenses-compliance/check-licenses-compliance.py
+++ b/scripts/licenses-compliance/check-licenses-compliance.py
@@ -11,7 +11,7 @@
     "ISC License (ISCL)",
     "Public Domain",
     "Python Software Foundation License",
-    # "Mozilla Public License 2.0 (MPL 2.0)",
+    "Mozilla Public License 2.0 (MPL 2.0)",
     "MIT License",
     "The Unlicense (Unlicense)",
 ]
@@ -29,6 +29,10 @@ def check_license_compliance(
             # note that "License" can be a comma separated list of licenses
             licenses = row["License"]
             for license in licenses.split(";"):
+                # edge case(s) like `tiktoken`
+                if "\n" in license:
+                    license = license.split("\n")[0]
+
                 license = license.strip()
                 if license not in allowed_licenses:
                     incompliance_items.append((name, licenses))

From 5c0fe86c3537a7884dcc5559bf181ae84d7553d7 Mon Sep 17 00:00:00 2001
From: Zhengfei Wang <wangzhengfei.chn@outlook.com>
Date: Tue, 9 Jul 2024 08:39:32 +0000
Subject: [PATCH 12/25] install all extras

---
 .github/workflows/license-compliance.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/license-compliance.yml b/.github/workflows/license-compliance.yml
index 7b1eece93e2..29c7471a55f 100644
--- a/.github/workflows/license-compliance.yml
+++ b/.github/workflows/license-compliance.yml
@@ -9,12 +9,12 @@ jobs:
     - uses: actions/checkout@v4
     - uses: actions/setup-python@v5
       with:
-        python-version: '3.11'
+        python-version: '3.12'
     - name: install pip-licenses
       run: pip install pip-licenses
     - name: check licenses
       run: |
-        pip install .
+        pip install .[executor-service,azureml-serving]
         pip-licenses --from=mixed --format=csv --output-file=LICENSES.csv
         cat LICENSES.csv
         python ../../scripts/licenses-compliance/check-licenses-compliance.py LICENSES.csv

From a75d8e593afb8beea34c2c4ec9d6828b31ee1ece Mon Sep 17 00:00:00 2001
From: Zhengfei Wang <wangzhengfei.chn@outlook.com>
Date: Tue, 9 Jul 2024 08:42:21 +0000
Subject: [PATCH 13/25] add pip list & exception

---
 .github/workflows/license-compliance.yml                 | 1 +
 scripts/licenses-compliance/check-licenses-compliance.py | 1 +
 2 files changed, 2 insertions(+)

diff --git a/.github/workflows/license-compliance.yml b/.github/workflows/license-compliance.yml
index 29c7471a55f..2b0a2e4d30d 100644
--- a/.github/workflows/license-compliance.yml
+++ b/.github/workflows/license-compliance.yml
@@ -15,6 +15,7 @@ jobs:
     - name: check licenses
       run: |
         pip install .[executor-service,azureml-serving]
+        pip list
         pip-licenses --from=mixed --format=csv --output-file=LICENSES.csv
         cat LICENSES.csv
         python ../../scripts/licenses-compliance/check-licenses-compliance.py LICENSES.csv
diff --git a/scripts/licenses-compliance/check-licenses-compliance.py b/scripts/licenses-compliance/check-licenses-compliance.py
index 1354c591ded..27a373b8b89 100644
--- a/scripts/licenses-compliance/check-licenses-compliance.py
+++ b/scripts/licenses-compliance/check-licenses-compliance.py
@@ -12,6 +12,7 @@
     "Public Domain",
     "Python Software Foundation License",
     "Mozilla Public License 2.0 (MPL 2.0)",
+    "3-Clause BSD License",
     "MIT License",
     "The Unlicense (Unlicense)",
 ]

From 4ba33d09c3de10fe525d22ececebe4a8bd37571b Mon Sep 17 00:00:00 2001
From: Zhengfei Wang <wangzhengfei.chn@outlook.com>
Date: Thu, 11 Jul 2024 05:17:39 +0000
Subject: [PATCH 14/25] add CG pipeline

---
 .github/pipelines/component-governance.yml | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 .github/pipelines/component-governance.yml

diff --git a/.github/pipelines/component-governance.yml b/.github/pipelines/component-governance.yml
new file mode 100644
index 00000000000..9218d8cfe80
--- /dev/null
+++ b/.github/pipelines/component-governance.yml
@@ -0,0 +1,9 @@
+
+name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.r)
+
+pool:
+  name: promptflow-1ES-win
+
+steps:
+- task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
+  displayName: 'Component Detection'

From db84bbd2548ae71c1a1c21a7f4ae67790e687fe4 Mon Sep 17 00:00:00 2001
From: Zhengfei Wang <wangzhengfei.chn@outlook.com>
Date: Thu, 11 Jul 2024 05:25:38 +0000
Subject: [PATCH 15/25] add CG to compliance check

---
 .github/pipelines/compliance_check.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/pipelines/compliance_check.yml b/.github/pipelines/compliance_check.yml
index b6152c9884c..d8ebcaee963 100644
--- a/.github/pipelines/compliance_check.yml
+++ b/.github/pipelines/compliance_check.yml
@@ -22,6 +22,9 @@ pool:
 steps:
 - checkout: self
 
+- task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
+  displayName: 'Component Detection'
+
 - task: PowerShell@2
   inputs:
     targetType: inline

From bd2f848b80b0c4b43e535bccb1b35ff30a92394f Mon Sep 17 00:00:00 2001
From: Zhengfei Wang <wangzhengfei.chn@outlook.com>
Date: Thu, 11 Jul 2024 06:07:08 +0000
Subject: [PATCH 16/25] add scan path

---
 .github/pipelines/component-governance.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/pipelines/component-governance.yml b/.github/pipelines/component-governance.yml
index 9218d8cfe80..e2c232906ba 100644
--- a/.github/pipelines/component-governance.yml
+++ b/.github/pipelines/component-governance.yml
@@ -6,4 +6,6 @@ pool:
 
 steps:
 - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
+  inputs:
+    sourceScanPath: ../../src/
   displayName: 'Component Detection'

From 25f9e79cb68972ea65984b9cb82527f0a021ec1b Mon Sep 17 00:00:00 2001
From: Zhengfei Wang <wangzhengfei.chn@outlook.com>
Date: Thu, 11 Jul 2024 06:07:49 +0000
Subject: [PATCH 17/25] update CG for scan path

---
 .github/pipelines/compliance_check.yml     |  8 +++++---
 .github/pipelines/component-governance.yml | 11 -----------
 2 files changed, 5 insertions(+), 14 deletions(-)
 delete mode 100644 .github/pipelines/component-governance.yml

diff --git a/.github/pipelines/compliance_check.yml b/.github/pipelines/compliance_check.yml
index d8ebcaee963..c6dc0dd197c 100644
--- a/.github/pipelines/compliance_check.yml
+++ b/.github/pipelines/compliance_check.yml
@@ -22,9 +22,6 @@ pool:
 steps:
 - checkout: self
 
-- task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
-  displayName: 'Component Detection'
-
 - task: PowerShell@2
   inputs:
     targetType: inline
@@ -74,3 +71,8 @@ steps:
   displayName: 'CredScan'
   inputs:
     scanFolder: '$(sourceLocation)'
+
+- task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
+  inputs:
+    sourceScanPath: ../../src/
+  displayName: 'Component Detection'
diff --git a/.github/pipelines/component-governance.yml b/.github/pipelines/component-governance.yml
deleted file mode 100644
index e2c232906ba..00000000000
--- a/.github/pipelines/component-governance.yml
+++ /dev/null
@@ -1,11 +0,0 @@
-
-name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.r)
-
-pool:
-  name: promptflow-1ES-win
-
-steps:
-- task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
-  inputs:
-    sourceScanPath: ../../src/
-  displayName: 'Component Detection'

From fa0fd0e856b24ce2d311c1734076fa85d59cb1f1 Mon Sep 17 00:00:00 2001
From: Zhengfei Wang <wangzhengfei.chn@outlook.com>
Date: Thu, 11 Jul 2024 06:08:27 +0000
Subject: [PATCH 18/25] use macro

---
 .github/pipelines/compliance_check.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/pipelines/compliance_check.yml b/.github/pipelines/compliance_check.yml
index c6dc0dd197c..2f9ddf5a57e 100644
--- a/.github/pipelines/compliance_check.yml
+++ b/.github/pipelines/compliance_check.yml
@@ -74,5 +74,5 @@ steps:
 
 - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
   inputs:
-    sourceScanPath: ../../src/
+    sourceScanPath: '$(sourceLocation)\src'
   displayName: 'Component Detection'

From 3e9d1a4f2ddbe0c8ba050835e80d79da3fc94b9b Mon Sep 17 00:00:00 2001
From: Zhengfei Wang <wangzhengfei.chn@outlook.com>
Date: Thu, 11 Jul 2024 06:11:20 +0000
Subject: [PATCH 19/25] remove script

---
 .github/workflows/license-compliance.yml      | 22 --------
 .../check-licenses-compliance.py              | 55 -------------------
 2 files changed, 77 deletions(-)
 delete mode 100644 .github/workflows/license-compliance.yml
 delete mode 100644 scripts/licenses-compliance/check-licenses-compliance.py

diff --git a/.github/workflows/license-compliance.yml b/.github/workflows/license-compliance.yml
deleted file mode 100644
index 2b0a2e4d30d..00000000000
--- a/.github/workflows/license-compliance.yml
+++ /dev/null
@@ -1,22 +0,0 @@
-name: "License Compliance"
-on:
-- pull_request
-
-jobs:
-  promptflow-core:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v4
-    - uses: actions/setup-python@v5
-      with:
-        python-version: '3.12'
-    - name: install pip-licenses
-      run: pip install pip-licenses
-    - name: check licenses
-      run: |
-        pip install .[executor-service,azureml-serving]
-        pip list
-        pip-licenses --from=mixed --format=csv --output-file=LICENSES.csv
-        cat LICENSES.csv
-        python ../../scripts/licenses-compliance/check-licenses-compliance.py LICENSES.csv
-      working-directory: ${{ github.workspace }}/src/promptflow-core
diff --git a/scripts/licenses-compliance/check-licenses-compliance.py b/scripts/licenses-compliance/check-licenses-compliance.py
deleted file mode 100644
index 27a373b8b89..00000000000
--- a/scripts/licenses-compliance/check-licenses-compliance.py
+++ /dev/null
@@ -1,55 +0,0 @@
-import csv
-import sys
-import typing
-from pathlib import Path
-
-# list of licenses compatible with MIT
-allowed_licenses = [
-    "Apache Software License",
-    "BSD License",
-    "GNU General Public License (GPL)",
-    "ISC License (ISCL)",
-    "Public Domain",
-    "Python Software Foundation License",
-    "Mozilla Public License 2.0 (MPL 2.0)",
-    "3-Clause BSD License",
-    "MIT License",
-    "The Unlicense (Unlicense)",
-]
-
-
-def check_license_compliance(
-    licenses_file: str,
-    allowed_licenses: typing.List[str],
-) -> typing.List[str]:
-    incompliance_items = list()
-    with open(licenses_file, mode="r") as file:
-        csv_reader = csv.DictReader(file)
-        for row in csv_reader:
-            name = row["Name"]
-            # note that "License" can be a comma separated list of licenses
-            licenses = row["License"]
-            for license in licenses.split(";"):
-                # edge case(s) like `tiktoken`
-                if "\n" in license:
-                    license = license.split("\n")[0]
-
-                license = license.strip()
-                if license not in allowed_licenses:
-                    incompliance_items.append((name, licenses))
-        return incompliance_items
-
-
-def main(licenses_file: Path):
-    incompliance_items = check_license_compliance(licenses_file, allowed_licenses)
-    if len(incompliance_items) > 0:
-        print("found dependencies with licenses incompliance with MIT license:")
-        for name, license in incompliance_items:
-            print(f"- {name}: {license}")
-        raise Exception("found dependencies with licenses incompliance with MIT license.")
-    else:
-        print("all dependencies are compliance with MIT license.")
-
-
-if __name__ == "__main__":
-    main(licenses_file=Path(sys.argv[1]).resolve().absolute())

From 615c2c466bd4dbff5027efc03f1318c50813ca06 Mon Sep 17 00:00:00 2001
From: Zhengfei Wang <wangzhengfei.chn@outlook.com>
Date: Thu, 11 Jul 2024 06:30:04 +0000
Subject: [PATCH 20/25] add pre-process step

---
 .github/pipelines/compliance_check.yml | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/.github/pipelines/compliance_check.yml b/.github/pipelines/compliance_check.yml
index 2f9ddf5a57e..f2c47b38ed0 100644
--- a/.github/pipelines/compliance_check.yml
+++ b/.github/pipelines/compliance_check.yml
@@ -72,6 +72,28 @@ steps:
   inputs:
     scanFolder: '$(sourceLocation)'
 
+# according to doc: https://github.com/microsoft/component-detection/blob/main/docs/feature-overview.md
+# CG does not support pyproject.toml, and will scan all requirements.txt
+# so we need this step to:
+#   1) create some requirements.txt from pyproject.toml
+#   2) remove unexpected requirements.txt
+- task: PowerShell@2
+  inputs:
+    targetType: inline
+    script: |
+      pip install toml-to-requirements
+      Write-Host "Generating requirements.txt from pyproject.toml..."
+      Write-Host "##########################################"
+      Write-Host "- promptflow-tracing:"
+      Set-Location "$(sourceLocation)/src/promptflow-tracing/"
+      toml-to-req --toml-file pyproject.toml --poetry
+      Get-ChildItem
+      Get-Content requirements.txt
+
+      Write-Host "Removing requirements.txt..."
+      # Remove-Item -Recurse -Force $(sourceLocation)/promptflow/test
+  displayName: 'Preprocess before CG'
+
 - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
   inputs:
     sourceScanPath: '$(sourceLocation)\src'

From 2d3b772e8ce11fe4176da0c24839ed1176c2c76a Mon Sep 17 00:00:00 2001
From: Zhengfei Wang <wangzhengfei.chn@outlook.com>
Date: Thu, 11 Jul 2024 06:48:57 +0000
Subject: [PATCH 21/25] add more toml to req and remove some in test configs

---
 .github/pipelines/compliance_check.yml | 22 ++++++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/.github/pipelines/compliance_check.yml b/.github/pipelines/compliance_check.yml
index f2c47b38ed0..5a4ddcf8b8c 100644
--- a/.github/pipelines/compliance_check.yml
+++ b/.github/pipelines/compliance_check.yml
@@ -86,12 +86,30 @@ steps:
       Write-Host "##########################################"
       Write-Host "- promptflow-tracing:"
       Set-Location "$(sourceLocation)/src/promptflow-tracing/"
+      Get-Content pyproject.toml
+      toml-to-req --toml-file pyproject.toml --poetry
+      Get-Content requirements.txt
+      Write-Host "##########################################"
+      Write-Host "- promptflow-core:"
+      Set-Location "$(sourceLocation)/src/promptflow-core/"
+      Get-Content pyproject.toml
+      toml-to-req --toml-file pyproject.toml --poetry
+      Get-Content requirements.txt
+      Write-Host "##########################################"
+      Write-Host "- promptflow-devkit:"
+      Set-Location "$(sourceLocation)/src/promptflow-devkit/"
+      Get-Content pyproject.toml
+      toml-to-req --toml-file pyproject.toml --poetry
+      Get-Content requirements.txt
+      Write-Host "##########################################"
+      Write-Host "- promptflow-azure:"
+      Set-Location "$(sourceLocation)/src/promptflow-azure/"
+      Get-Content pyproject.toml
       toml-to-req --toml-file pyproject.toml --poetry
-      Get-ChildItem
       Get-Content requirements.txt
 
       Write-Host "Removing requirements.txt..."
-      # Remove-Item -Recurse -Force $(sourceLocation)/promptflow/test
+      Get-ChildItem -Path "$(sourceLocation)/src/promptflow/tests" -Recurse -Include requirements.txt | Where-Object { $_.FullName -match "promptflow/test" } | ForEach-Object { Remove-Item -Path $_.FullName -Force }
   displayName: 'Preprocess before CG'
 
 - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0

From e3b33c9397065cbc57f4ab0e7f7bfbfdc5f1b76b Mon Sep 17 00:00:00 2001
From: Zhengfei Wang <wangzhengfei.chn@outlook.com>
Date: Thu, 11 Jul 2024 07:19:57 +0000
Subject: [PATCH 22/25] no cat for toml and remove req in tests

---
 .github/pipelines/compliance_check.yml | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/.github/pipelines/compliance_check.yml b/.github/pipelines/compliance_check.yml
index 5a4ddcf8b8c..035e8382440 100644
--- a/.github/pipelines/compliance_check.yml
+++ b/.github/pipelines/compliance_check.yml
@@ -86,30 +86,29 @@ steps:
       Write-Host "##########################################"
       Write-Host "- promptflow-tracing:"
       Set-Location "$(sourceLocation)/src/promptflow-tracing/"
-      Get-Content pyproject.toml
       toml-to-req --toml-file pyproject.toml --poetry
       Get-Content requirements.txt
       Write-Host "##########################################"
       Write-Host "- promptflow-core:"
       Set-Location "$(sourceLocation)/src/promptflow-core/"
-      Get-Content pyproject.toml
       toml-to-req --toml-file pyproject.toml --poetry
       Get-Content requirements.txt
       Write-Host "##########################################"
       Write-Host "- promptflow-devkit:"
       Set-Location "$(sourceLocation)/src/promptflow-devkit/"
-      Get-Content pyproject.toml
       toml-to-req --toml-file pyproject.toml --poetry
       Get-Content requirements.txt
       Write-Host "##########################################"
       Write-Host "- promptflow-azure:"
       Set-Location "$(sourceLocation)/src/promptflow-azure/"
-      Get-Content pyproject.toml
       toml-to-req --toml-file pyproject.toml --poetry
       Get-Content requirements.txt
+      Write-Host "##########################################"
 
       Write-Host "Removing requirements.txt..."
-      Get-ChildItem -Path "$(sourceLocation)/src/promptflow/tests" -Recurse -Include requirements.txt | Where-Object { $_.FullName -match "promptflow/test" } | ForEach-Object { Remove-Item -Path $_.FullName -Force }
+      Set-Location "$(sourceLocation)/src/"
+      Write-Host "- src/promptflow/tests/*:"
+      Get-ChildItem -Path "src/promptflow/tests" -Recurse -Include requirements.txt | ForEach-Object { Remove-Item -Path $_.FullName -Force }
   displayName: 'Preprocess before CG'
 
 - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0

From 6593d74a649c12ce4d2911a4753818126055daa7 Mon Sep 17 00:00:00 2001
From: Zhengfei Wang <wangzhengfei.chn@outlook.com>
Date: Thu, 11 Jul 2024 07:33:16 +0000
Subject: [PATCH 23/25] fix: remove src

---
 .github/pipelines/compliance_check.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/pipelines/compliance_check.yml b/.github/pipelines/compliance_check.yml
index 035e8382440..1c475532547 100644
--- a/.github/pipelines/compliance_check.yml
+++ b/.github/pipelines/compliance_check.yml
@@ -106,7 +106,7 @@ steps:
       Write-Host "##########################################"
 
       Write-Host "Removing requirements.txt..."
-      Set-Location "$(sourceLocation)/src/"
+      Set-Location "$(sourceLocation)/"
       Write-Host "- src/promptflow/tests/*:"
       Get-ChildItem -Path "src/promptflow/tests" -Recurse -Include requirements.txt | ForEach-Object { Remove-Item -Path $_.FullName -Force }
   displayName: 'Preprocess before CG'

From 0923bce75142173e89224ca68f536185e9f37a83 Mon Sep 17 00:00:00 2001
From: Zhengfei Wang <wangzhengfei.chn@outlook.com>
Date: Thu, 11 Jul 2024 07:48:16 +0000
Subject: [PATCH 24/25] test: add a GPLv3 dep

---
 src/promptflow-azure/pyproject.toml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/promptflow-azure/pyproject.toml b/src/promptflow-azure/pyproject.toml
index 4febc3820af..1c6ec927cb4 100644
--- a/src/promptflow-azure/pyproject.toml
+++ b/src/promptflow-azure/pyproject.toml
@@ -46,6 +46,7 @@ azure-ai-ml = ">=1.14.0,<2.0.0"
 azure-cosmos = ">=4.5.1,<5.0.0"  # used to upload trace to cloud
 pyjwt = ">=2.4.0,<3.0.0"  # requirement of control plane SDK
 promptflow-devkit = "<2.0.0"
+flirextractor = ">=1.0.2"
 
 [tool.poetry.group.dev.dependencies]
 pre-commit = "*"

From 24c21d997975481304ab07a2c5262faee32a87f3 Mon Sep 17 00:00:00 2001
From: Zhengfei Wang <wangzhengfei.chn@outlook.com>
Date: Thu, 11 Jul 2024 07:58:34 +0000
Subject: [PATCH 25/25] Revert "test: add a GPLv3 dep"

This reverts commit 0923bce75142173e89224ca68f536185e9f37a83.
---
 src/promptflow-azure/pyproject.toml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/promptflow-azure/pyproject.toml b/src/promptflow-azure/pyproject.toml
index 1c6ec927cb4..4febc3820af 100644
--- a/src/promptflow-azure/pyproject.toml
+++ b/src/promptflow-azure/pyproject.toml
@@ -46,7 +46,6 @@ azure-ai-ml = ">=1.14.0,<2.0.0"
 azure-cosmos = ">=4.5.1,<5.0.0"  # used to upload trace to cloud
 pyjwt = ">=2.4.0,<3.0.0"  # requirement of control plane SDK
 promptflow-devkit = "<2.0.0"
-flirextractor = ">=1.0.2"
 
 [tool.poetry.group.dev.dependencies]
 pre-commit = "*"