Skip to content

Commit

Permalink
Switch to dlt for git
Browse files Browse the repository at this point in the history
maximearmstrong committed Apr 25, 2024
1 parent 3989b67 commit 17abf3d
Showing 2 changed files with 4 additions and 17 deletions.
6 changes: 0 additions & 6 deletions .github/workflows/deploy.yml
Original file line number Diff line number Diff line change
@@ -41,11 +41,6 @@ jobs:
with:
ref: ${{ github.head_ref }}
path: project-repo
- name: Install git
if: steps.prerun.outputs.result == 'pex-deploy'
run: |
sudo apt-get install git
shell: bash

- name: Python Executable Deploy
if: steps.prerun.outputs.result == 'pex-deploy'
@@ -56,7 +51,6 @@ jobs:
python_version: "${{ env.PYTHON_VERSION }}"
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GIT_PYTHON_GIT_EXECUTABLE: "/usr/bin/git"

dagster_cloud_docker_deploy:
name: Docker Deploy
15 changes: 4 additions & 11 deletions with_openai/utils.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
import os
import pathlib
import tempfile
import subprocess

import requests
from dlt.common import git
from langchain.docstore.document import Document


@@ -20,18 +20,11 @@ def get_wiki_data(title, first_paragraph_only):

def get_github_docs(repo_owner, repo_name, category):
with tempfile.TemporaryDirectory() as d:
#repo = Repo.clone_from(f"https://github.com/{repo_owner}/{repo_name}.git", d, depth=1)
#git_sha = repo.rev_parse("HEAD").hexsha
subprocess.check_call(
f"/usr/bin/git clone --depth 1 https://github.com/{repo_owner}/{repo_name}.git .",
cwd=d,
shell=True,
)
git_sha = (
subprocess.check_output("/usr/bin/git rev-parse HEAD", shell=True, cwd=d).decode("utf-8").strip()
)
repo = git.clone_repo(f"https://github.com/{repo_owner}/{repo_name}.git", d, with_git_command="--depth=1")
git_sha = repo.rev_parse("HEAD").hexsha
docs_path = pathlib.Path(os.path.join(d, "docs/content", category))
markdown_files = list(docs_path.glob("*.md*")) + list(docs_path.glob("*/*.md*"))
print(markdown_files)
for markdown_file in markdown_files:
with open(markdown_file, "r") as f:
relative_path = markdown_file.relative_to(d)

0 comments on commit 17abf3d

Please sign in to comment.