Skip to content

Commit b1e2e38

Browse files
codablockcodablock
authored
Merge pull request #43 from kluctl/feat-external-python
feat: Support non-embedded python via the same interface
2 parents 1ba1472 + 421798e commit b1e2e38

File tree

12 files changed

+181
-77
lines changed

12 files changed

+181
-77
lines changed

‎.github/workflows/release.yml

+22-12
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,13 @@ on:
1111
env:
1212
PYTHON_STANDALONE_VERSIONS: |
1313
[
14-
"20240224"
14+
"20240415"
1515
]
1616
PYTHON_VERSIONS: |
1717
[
18-
"3.10.13",
19-
"3.11.8",
20-
"3.12.2"
18+
"3.10.14",
19+
"3.11.9",
20+
"3.12.3"
2121
]
2222
2323
jobs:
@@ -39,14 +39,24 @@ jobs:
3939
pythonStandaloneVersion: ${{ fromJSON(needs.build-matrix.outputs.PYTHON_STANDALONE_VERSIONS) }}
4040
pythonVersion: ${{ fromJSON(needs.build-matrix.outputs.PYTHON_VERSIONS) }}
4141
fail-fast: false
42-
runs-on: ubuntu-20.04
42+
runs-on: ubuntu-22.04
4343
steps:
44-
- name: checkout
44+
- name: clone
4545
run: |
4646
# can't use actions/checkout here as transferring the shallow clone fails when using upload-/download-artifact
47-
git clone https://token:[email protected]/$GITHUB_REPOSITORY . --depth=1
48-
env:
49-
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
47+
git clone https://github.com/$GITHUB_REPOSITORY . --depth=1
48+
- name: checkout PR
49+
if: ${{ github.event_name == 'pull_request' }}
50+
run: |
51+
echo fetching pull/${{ github.ref_name }}
52+
git fetch origin pull/${{ github.ref_name }}:pr --depth=1
53+
git checkout pr
54+
- name: checkout branch
55+
if: ${{ github.event_name == 'push' }}
56+
run: |
57+
echo fetching ${{ github.ref_name }}
58+
git fetch origin ${{ github.ref_name }} --depth=1
59+
git checkout ${{ github.ref_name }}
5060
- name: Set up Go
5161
uses: actions/setup-go@v5
5262
with:
@@ -83,8 +93,8 @@ jobs:
8393
strategy:
8494
matrix:
8595
os:
86-
- ubuntu-20.04
87-
- macos-11
96+
- ubuntu-22.04
97+
- macos-12
8898
- windows-2019
8999
pythonStandaloneVersion: ${{ fromJSON(needs.build-matrix.outputs.PYTHON_STANDALONE_VERSIONS) }}
90100
pythonVersion: ${{ fromJSON(needs.build-matrix.outputs.PYTHON_VERSIONS) }}
@@ -125,7 +135,7 @@ jobs:
125135
pythonStandaloneVersion: ${{ fromJSON(needs.build-matrix.outputs.PYTHON_STANDALONE_VERSIONS) }}
126136
pythonVersion: ${{ fromJSON(needs.build-matrix.outputs.PYTHON_VERSIONS) }}
127137
fail-fast: false
128-
runs-on: ubuntu-20.04
138+
runs-on: ubuntu-22.04
129139
if: ${{ github.event_name == 'push' && github.ref_name == 'main' }}
130140
permissions:
131141
contents: write

‎example/main.go

+4-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,10 @@ func main() {
1111
panic(err)
1212
}
1313

14-
cmd := ep.PythonCmd("-c", "print('hello')")
14+
cmd, err := ep.PythonCmd("-c", "print('hello')")
15+
if err != nil {
16+
panic(err)
17+
}
1518
cmd.Stdout = os.Stdout
1619
cmd.Stderr = os.Stderr
1720
err = cmd.Run()

‎go.mod

+2-2
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ go 1.19
44

55
require (
66
github.com/gobwas/glob v0.2.3
7-
github.com/klauspost/compress v1.17.8
7+
github.com/klauspost/compress v1.17.9
88
github.com/rogpeppe/go-internal v1.12.0
99
github.com/sirupsen/logrus v1.9.3
1010
github.com/stretchr/testify v1.9.0
@@ -14,6 +14,6 @@ require (
1414
require (
1515
github.com/davecgh/go-spew v1.1.1 // indirect
1616
github.com/pmezard/go-difflib v1.0.0 // indirect
17-
golang.org/x/sys v0.15.0 // indirect
17+
golang.org/x/sys v0.21.0 // indirect
1818
gopkg.in/yaml.v3 v3.0.1 // indirect
1919
)

‎go.sum

+4
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLA
77
github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
88
github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=
99
github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
10+
github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
11+
github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
1012
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
1113
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
1214
github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
@@ -24,6 +26,8 @@ golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
2426
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
2527
golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
2628
golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
29+
golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
30+
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
2731
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
2832
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
2933
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

‎hack/build-tag.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ go run ./pip/generate
3535
TAG=v0.0.0-$PYTHON_VERSION-$PYTHON_STANDALONE_VERSION-$BUILD_NUM
3636

3737
echo "checking out temporary branch"
38-
git checkout $(git rev-parse HEAD)
38+
git checkout --detach
3939
git add -f python/internal/data
4040
git add -f pip/internal/data
4141
git commit -m "added python $PYTHON_VERSION from python-standalone $PYTHON_STANDALONE_VERSION"

‎pip/embed_pip_packages.go

+5-2
Original file line numberDiff line numberDiff line change
@@ -109,10 +109,13 @@ func pipInstall(ep *python.EmbeddedPython, requirementsFile string, platforms []
109109
args = append(args, "--only-binary=:all:")
110110
}
111111

112-
cmd := ep.PythonCmd(args...)
112+
cmd, err := ep.PythonCmd(args...)
113+
if err != nil {
114+
return err
115+
}
113116
cmd.Stdout = os.Stdout
114117
cmd.Stderr = os.Stderr
115-
err := cmd.Run()
118+
err = cmd.Run()
116119
if err != nil {
117120
return err
118121
}

‎pip/generate/main.go

+5-2
Original file line numberDiff line numberDiff line change
@@ -33,10 +33,13 @@ func bootstrapPip(ep *python.EmbeddedPython) {
3333
getPip := downloadGetPip()
3434
defer os.Remove(getPip)
3535

36-
cmd := ep.PythonCmd(getPip)
36+
cmd, err := ep.PythonCmd(getPip)
37+
if err != nil {
38+
panic(err)
39+
}
3740
cmd.Stdout = os.Stdout
3841
cmd.Stderr = os.Stderr
39-
err := cmd.Run()
42+
err = cmd.Run()
4043
if err != nil {
4144
panic(err)
4245
}

‎python/embedded_python.go

+5-50
Original file line numberDiff line numberDiff line change
@@ -4,17 +4,11 @@ import (
44
"fmt"
55
"github.com/kluctl/go-embed-python/embed_util"
66
"github.com/kluctl/go-embed-python/python/internal/data"
7-
"os"
8-
"os/exec"
9-
"path/filepath"
10-
"runtime"
11-
"strings"
127
)
138

149
type EmbeddedPython struct {
1510
e *embed_util.EmbeddedFiles
16-
17-
pythonPath []string
11+
*Python
1812
}
1913

2014
// NewEmbeddedPython creates a new EmbeddedPython instance. The embedded source code and python binaries are
@@ -26,7 +20,8 @@ func NewEmbeddedPython(name string) (*EmbeddedPython, error) {
2620
return nil, err
2721
}
2822
return &EmbeddedPython{
29-
e: e,
23+
e: e,
24+
Python: NewPython(WithPythonHome(e.GetExtractedPath())),
3025
}, nil
3126
}
3227

@@ -36,7 +31,8 @@ func NewEmbeddedPythonWithTmpDir(tmpDir string, withHashInDir bool) (*EmbeddedPy
3631
return nil, err
3732
}
3833
return &EmbeddedPython{
39-
e: e,
34+
e: e,
35+
Python: NewPython(WithPythonHome(e.GetExtractedPath())),
4036
}, nil
4137
}
4238

@@ -47,44 +43,3 @@ func (ep *EmbeddedPython) Cleanup() error {
4743
func (ep *EmbeddedPython) GetExtractedPath() string {
4844
return ep.e.GetExtractedPath()
4945
}
50-
51-
func (ep *EmbeddedPython) GetBinPath() string {
52-
if runtime.GOOS == "windows" {
53-
return ep.GetExtractedPath()
54-
} else {
55-
return filepath.Join(ep.GetExtractedPath(), "bin")
56-
}
57-
}
58-
59-
func (ep *EmbeddedPython) GetExePath() string {
60-
suffix := ""
61-
if runtime.GOOS == "windows" {
62-
suffix = ".exe"
63-
} else {
64-
suffix = "3"
65-
}
66-
return filepath.Join(ep.GetBinPath(), "python"+suffix)
67-
}
68-
69-
func (ep *EmbeddedPython) AddPythonPath(p string) {
70-
ep.pythonPath = append(ep.pythonPath, p)
71-
}
72-
73-
func (ep *EmbeddedPython) PythonCmd(args ...string) *exec.Cmd {
74-
return ep.PythonCmd2(args)
75-
}
76-
77-
func (ep *EmbeddedPython) PythonCmd2(args []string) *exec.Cmd {
78-
exePath := ep.GetExePath()
79-
80-
cmd := exec.Command(exePath, args...)
81-
cmd.Env = os.Environ()
82-
cmd.Env = append(cmd.Env, fmt.Sprintf("PYTHONHOME=%s", ep.GetExtractedPath()))
83-
84-
if len(ep.pythonPath) != 0 {
85-
pythonPathEnv := fmt.Sprintf("PYTHONPATH=%s", strings.Join(ep.pythonPath, string(os.PathListSeparator)))
86-
cmd.Env = append(cmd.Env, pythonPathEnv)
87-
}
88-
89-
return cmd
90-
}

‎python/embedded_python_test.go

+4-4
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,10 @@ func TestEmbeddedPython(t *testing.T) {
1717
defer ep.Cleanup()
1818
path := ep.GetExtractedPath()
1919
assert.NotEqual(t, path, "")
20-
pexe := ep.GetExePath()
20+
pexe, _ := ep.GetExePath()
2121
assert.True(t, internal.Exists(pexe))
2222

23-
cmd := ep.PythonCmd("-c", "print('test test')")
23+
cmd, _ := ep.PythonCmd("-c", "print('test test')")
2424
stdout, err := cmd.StdoutPipe()
2525
assert.NoError(t, err)
2626
defer stdout.Close()
@@ -61,10 +61,10 @@ print("platform.processor=" + platform.processor())
6161
defer ep.Cleanup()
6262
path := ep.GetExtractedPath()
6363
assert.NotEqual(t, path, "")
64-
pexe := ep.GetExePath()
64+
pexe, _ := ep.GetExePath()
6565
assert.True(t, internal.Exists(pexe))
6666

67-
cmd := ep.PythonCmd("-c", getSystemInfo)
67+
cmd, _ := ep.PythonCmd("-c", getSystemInfo)
6868
stdout, _ := cmd.StdoutPipe()
6969
stderr, _ := cmd.StderrPipe()
7070
defer stdout.Close()

‎python/generate/main.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -81,10 +81,10 @@ func main() {
8181
}
8282

8383
jobs := []job{
84-
{"linux", "amd64", "unknown-linux-gnu-lto-full", keepNixPatterns},
84+
{"linux", "amd64", "unknown-linux-gnu-pgo+lto-full", keepNixPatterns},
8585
{"linux", "arm64", "unknown-linux-gnu-lto-full", keepNixPatterns},
86-
{"darwin", "amd64", "apple-darwin-lto-full", keepNixPatterns},
87-
{"darwin", "arm64", "apple-darwin-lto-full", keepNixPatterns},
86+
{"darwin", "amd64", "apple-darwin-pgo+lto-full", keepNixPatterns},
87+
{"darwin", "arm64", "apple-darwin-pgo+lto-full", keepNixPatterns},
8888
{"windows", "amd64", "pc-windows-msvc-shared-pgo-full", keepWinPatterns},
8989
}
9090
for _, j := range jobs {

‎python/python.go

+93
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
package python
2+
3+
import (
4+
"fmt"
5+
"os"
6+
"os/exec"
7+
"path/filepath"
8+
"runtime"
9+
"strings"
10+
)
11+
12+
type Python struct {
13+
pythonHome string
14+
pythonPath []string
15+
}
16+
17+
type PythonOpt func(o *Python)
18+
19+
func WithPythonHome(home string) PythonOpt {
20+
return func(o *Python) {
21+
o.pythonHome = home
22+
}
23+
}
24+
25+
func NewPython(opts ...PythonOpt) *Python {
26+
ep := &Python{}
27+
28+
for _, o := range opts {
29+
o(ep)
30+
}
31+
32+
return ep
33+
}
34+
35+
func (ep *Python) GetExeName() string {
36+
suffix := ""
37+
if runtime.GOOS == "windows" {
38+
suffix = ".exe"
39+
} else {
40+
suffix = "3"
41+
}
42+
return "python" + suffix
43+
}
44+
45+
func (ep *Python) GetExePath() (string, error) {
46+
if ep.pythonHome == "" {
47+
p, err := exec.LookPath(ep.GetExeName())
48+
if err != nil {
49+
return "", fmt.Errorf("failed to determine %s path: %w", ep.GetExeName(), err)
50+
}
51+
return p, nil
52+
} else {
53+
var p string
54+
if runtime.GOOS == "windows" {
55+
p = filepath.Join(ep.pythonHome, ep.GetExeName())
56+
} else {
57+
p = filepath.Join(ep.pythonHome, "bin", ep.GetExeName())
58+
}
59+
if _, err := os.Stat(p); err != nil {
60+
return "", fmt.Errorf("failed to determine %s path: %w", ep.GetExeName(), err)
61+
}
62+
return p, nil
63+
}
64+
}
65+
66+
func (ep *Python) AddPythonPath(p string) {
67+
ep.pythonPath = append(ep.pythonPath, p)
68+
}
69+
70+
func (ep *Python) PythonCmd(args ...string) (*exec.Cmd, error) {
71+
return ep.PythonCmd2(args)
72+
}
73+
74+
func (ep *Python) PythonCmd2(args []string) (*exec.Cmd, error) {
75+
exePath, err := ep.GetExePath()
76+
if err != nil {
77+
return nil, err
78+
}
79+
80+
cmd := exec.Command(exePath, args...)
81+
cmd.Env = os.Environ()
82+
83+
if ep.pythonHome != "" {
84+
cmd.Env = append(cmd.Env, fmt.Sprintf("PYTHONHOME=%s", ep.pythonHome))
85+
}
86+
87+
if len(ep.pythonPath) != 0 {
88+
pythonPathEnv := fmt.Sprintf("PYTHONPATH=%s", strings.Join(ep.pythonPath, string(os.PathListSeparator)))
89+
cmd.Env = append(cmd.Env, pythonPathEnv)
90+
}
91+
92+
return cmd, nil
93+
}

0 commit comments

Comments
 (0)