scraper.nim

import std/[os, json, strutils, strformat, options, algorithm]

let
    resolverJsonPath = fmt"{getAppDir()}/resolver.json"
    repoList = parseFile("repos.json")
    readmePath = fmt"{getAppDir()}/README.md"

var readmeText = """
<!-- THIS FILE IS AUTOMATICALLY GENERATED BY scraper.nim DON'T EDIT IT MANUALLY! -->

# Regolith Filter Resolver
This repository contains the default filter resolver for [Regolith](https://github.com/Bedrock-OSS/regolith). It is used to enable installing Regolith filters using their short names instead of full URLs. You can read more about resolvers in Regolith documentation.

## Adding Your Filters to the Resolver
You can add your own filters to the resolver by creating a pull request to this repository that adds your repository URL to the `repos.json` file. This will mark your repository as a known filter repository, and add all filters from there to the resolver.

## Known Repositories
You can use this list to discover new filters to use with Regolith, below you can find a list of known standard filters (from Bedrock-OSS) and community filters.
"""

var resolverJson = parseFile(resolverJsonPath)

type
    Filter = object
        name: string
        author: string
        description: string
        url: string
        versions: seq[string]

proc newFilter(name, author, description, url: string, versions: seq[string]): Filter =
    result.name = name
    result.author = author
    result.description = description
    result.url = url
    result.versions = versions

proc findRepo(author, repoName: string): string =
    for repo in repoList["known_repos"].to(seq[string]):
        if repo.contains(author) and repo.contains(repoName):
            return repo
    quit("Unable to run findRepo")

proc cloneRepos() =
    createDir("Repos")
    setCurrentDir("Repos")
    for repo in repoList["known_repos"].to(seq[string]):
        let cloneSplit = repo.split("/")
        let cloneName = cloneSplit[1] & "/" & cloneSplit[2]
        let cloneUrl = "http://" & repo
        if not dirExists(cloneName):
            discard execShellCmd(fmt"git clone {cloneUrl} {cloneName}")
    setCurrentDir(getAppDir())

proc readFilterDescription(filterJsonPath: string): string =
    try:
        let text = readFile(filterJsonPath)
        let jsonObject = parseJson(text)
        return jsonObject["description"].to(string)
    except:
        return ""

iterator getFilters(): Filter =
    let root = getCurrentDir()
    for owner in walkDir("Repos"):
        let user = splitPath(owner.path)[1]
        for repo in walkDir(owner.path):
            let repoName = splitPath(repo.path)[1]
            setCurrentDir(root & "/" & repo.path)
            discard execShellCmd("git tag > tags.txt")
            for filter in walkDir(getCurrentDir()):
                let filterName = splitPath(filter.path)[1]
                let filterJsonPath = filter.path & "/filter.json"
                if (fileExists(filterJsonPath)):
                    var versions: seq[string] = @["latest"]
                    let description = readFilterDescription(filterJsonPath)
                    let tagList = readFile("tags.txt").split("\n")
                    for tag in tagList:
                        if tag.contains(filterName & "-"):
                            let strVersion = tag.replace(filterName & "-", "")
                            versions.add(strVersion)
                    yield newFilter(filterName, user, description, findRepo(user, repoName), versions)
            setCurrentDir(root)


proc addFilter(f: Filter): Option[Filter] =
    ## Add a filter to the resolverJson. If filter with the same name but
    ## different URL already exists, return none, otherwise return some.
    if resolverJson["filters"].hasKey(f.name):
        if resolverJson["filters"][f.name]["url"].to(string) == f.url:
            echo fmt"Filter '{f.name}' already exists."
        else:
            echo fmt"Warning: Filter '{f.name}' already exists with different URL, skipping."
            return none(Filter)
    else:
        resolverJson["filters"][f.name] = %*{"url": f.url}
        echo fmt"Added filter {f.name}"
    return some(f)
cloneRepos()
var allFilters: seq[Filter] = @[]


# Collect the data about filters and expand the resolverJson
var standardFilters: seq[Filter] = @[]
var communityFilters: seq[Filter] = @[]

for onlineFilter in getFilters():
    let filterOption = onlineFilter.addFilter()
    if filterOption.isSome():
        let filter = filterOption.get()
        if filter.author == "Bedrock-OSS":
            standardFilters.add(filter)
        else:
            communityFilters.add(filter)

# Save the resolverJson
resolverJsonPath.writeFile(resolverJson.pretty())

# Sort the filters in the standardFilters and communityFilters to avoid
# making the README file change every time the script is run
proc sortFilters(a, b: Filter): int =
    let authorOrder = cmp(a.author.toLower(), b.author.toLower())
    if authorOrder != 0:
        return authorOrder
    return cmp(a.name.toLower(), b.name.toLower())

standardFilters.sort(sortFilters)
communityFilters.sort(sortFilters)

# Build the string for the README
readmeText.add("""
## Standard Filters

This list contains filters maintained by Bedrock-OSS.

| Filter | Description |
| ------ | ----------- |
""")
for filter in standardFilters:
    let description = filter.description.replace("|", "\\|")
    let name = filter.name.replace("|", "\\|")
    let url = filter.url.replace("|", "\\|")
    readmeText.add(fmt"| [{name}](https://{url}/tree/HEAD/{name}) | {description} |" & "\n")

readmeText.add("""
## Community Filters

This list contains filters created by the community.

> [!WARNING]
> The filters in this list are not maintained by Bedrock-OSS, use them at your own risk.

| Filter | Author |  Description |
| ------ | ------ | ------------ |
""")

for filter in communityFilters:
    let description = filter.description.replace("|", "\\|")
    let name = filter.name.replace("|", "\\|")
    let url = filter.url.replace("|", "\\|")
    let author = filter.author.replace("|", "\\|")
    readmeText.add(fmt"| [{name}](https://{url}/tree/HEAD/{name}) | {author} | {description} |" & "\n")

# Save the README
readmePath.writeFile(readmeText)

sleep(1500)