Add files for adding resources to existing datasets but simplify to c…

…reating new datasets without history

.config/coveragerc

@@ -0,0 +1,17 @@
+[run]
+source = src
+
+omit = */_version.py
+
+[report]
+exclude_also =
+    from ._version
+    def __repr__
+    if self.debug:
+    if settings.DEBUG
+    raise AssertionError
+    raise NotImplementedError
+    if 0:
+    if __name__ == .__main__.:
+    if TYPE_CHECKING:
+    @(abc\.)?abstractmethod

.config/pre-commit-config.yaml

@@ -0,0 +1,26 @@
+default_language_version:
+    python: python3.12
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-ast
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.7.0
+    hooks:
+      # Run the linter.
+      - id: ruff
+        args: [--config, .config/ruff.toml, --fix]
+      # Run the formatter.
+      - id: ruff-format
+        args: [--config, .config/ruff.toml]
+  - repo: https://github.com/astral-sh/uv-pre-commit
+    rev: 0.4.24
+    hooks:
+      # Run the pip compile
+      - id: pip-compile
+        name: pip-compile requirements.txt
+        files: pyproject.toml
+        args: [ pyproject.toml, --resolver=backtracking, --all-extras, --upgrade, -q, -o, requirements.txt ]

.config/pytest.ini

@@ -0,0 +1,4 @@
+[pytest]
+pythonpath = ../src
+addopts = "--color=yes"
+log_cli = 1

.config/ruff.toml

@@ -0,0 +1,16 @@
+line-length = 79
+exclude = ["_version.py"]
+
+[lint]
+# List of rules: https://docs.astral.sh/ruff/rules/
+select = [
+  "E",   # pycodestyle - default
+  "F",   # pyflakes - default
+  "I"    # isort
+]
+ignore = [
+    "E501" # Line too long
+]
+
+[lint.isort]
+known-local-folder = ["hdx"]

.github/workflows/publish.yaml

@@ -0,0 +1,34 @@
+name: Publish to PyPI
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Get history and tags for versioning to work
+      run: |
+        git fetch --prune --unshallow
+        git fetch --depth=1 origin +refs/tags/*:refs/tags/*
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.x'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+    - name: Install Hatch
+      uses: pypa/hatch@install
+    - name: Build with hatch
+      run: |
+        hatch build
+    - name: Publish with hatch
+      env:
+        HATCH_INDEX_USER: ${{secrets.HATCH_INDEX_USER}}
+        HATCH_INDEX_AUTH: ${{secrets.HATCH_INDEX_AUTH}}
+      run: |
+        hatch publish

.github/workflows/run-python-tests.yaml

@@ -0,0 +1,49 @@
+# This workflow will install Python dependencies, lint and run tests
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: Run tests
+
+on:
+  workflow_dispatch: # add run button in github
+  push:
+    branches-ignore:
+      - gh-pages
+      - 'dependabot/**'
+  pull_request:
+    branches-ignore:
+      - gh-pages
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.x'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+    - name: Install Hatch
+      uses: pypa/hatch@install
+    - name: Test with hatch/pytest
+      run: |
+        hatch test
+    - name: Check styling
+      if: always()
+      run: |
+        hatch fmt --check
+    - name: Publish Unit Test Results
+      uses: EnricoMi/publish-unit-test-result-action@v2
+      if: always()
+      with:
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+        junit_files: test-results.xml
+    - name: Publish in Coveralls
+      uses: coverallsapp/github-action@v2
+      with:
+        github-token: ${{ secrets.GITHUB_TOKEN }}
+        flag-name: tests
+        format: lcov

CHANGELOG.md

@@ -0,0 +1,11 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
+
+## [0.0.1] - 2024-03-13
+
+### Added
+
+-

CONTRIBUTING.md

@@ -0,0 +1,60 @@
+# Development
+
+## Environment
+
+Development is currently done using Python 3.12. We recommend using a virtual
+environment such as ``venv``:
+
+    python3.12 -m venv venv
+    source venv/bin/activate
+
+In your virtual environment, please install all packages for
+development by running:
+
+    pip install -r requirements.txt
+
+## Pre-Commit
+
+Also be sure to install `pre-commit`, which is run every time
+you make a git commit:
+
+    pre-commit install
+
+The configuration file for this project is in a
+non-standard location. Thus, you will need to edit your
+`.git/hooks/pre-commit` file to reflect this. Change
+the line that begins with `ARGS` to:
+
+    ARGS=(hook-impl --config=.config/pre-commit-config.yaml --hook-type=pre-commit)
+
+With pre-commit, all code is formatted according to
+[ruff]("https://github.com/charliermarsh/ruff") guidelines.
+
+To check if your changes pass pre-commit without committing, run:
+
+    pre-commit run --all-files --config=.config/pre-commit-config.yaml
+
+## Testing
+
+To run the tests and view coverage, execute:
+
+    pytest -c .config/pytest.ini --cov hdx --cov-config .config/coveragerc
+
+Follow the example set out already in ``documentation/main.md`` as you write the documentation.
+
+## Packages
+
+[pip-tools](https://github.com/jazzband/pip-tools) is used for
+package management.  If you’ve introduced a new package to the
+source code (i.e.anywhere in `src/`), please add it to the
+`project.dependencies` section of
+`pyproject.toml` with any known version constraints.
+
+For adding packages for testing or development, add them to
+the `test` or `dev` sections under `[project.optional-dependencies]`.
+
+Any changes to the dependencies will be automatically reflected in
+`requirements.txt` with `pre-commit`, but you can re-generate
+the file without committing by executing:
+
+    pre-commit run pip-compile --all-files --config=.config/pre-commit-config.yaml

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 UN-OCHA Humanitarian Data Exchange Project
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,9 @@
+# OPHI Pipeline
+
+[![Build Status](https://github.com/OCHA-DAP/hdx-scraper-ophi/actions/workflows/run-python-tests.yaml/badge.svg)](https://github.com/OCHA-DAP/hdx-scraper-ophi/actions/workflows/run-python-tests.yaml)
+[![Coverage Status](https://coveralls.io/repos/github/OCHA-DAP/hdx-scraper-ophi/badge.svg?branch=main&ts=1)](https://coveralls.io/github/OCHA-DAP/hdx-scraper-ophi?branch=main)
+[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
+
+This pipeline retrieves OPHI data
+
+## Running

pyproject.toml

@@ -0,0 +1,104 @@
+#########################
+# Project Configuration #
+#########################
+
+[build-system]
+requires = ["hatchling", "hatch-vcs"]
+build-backend = "hatchling.build"
+
+[project]
+name = "hdx-scraper-ophi"
+description = "OPHI Scraper"
+authors = [{ name = "Michael Rans", email = "[email protected]" }]
+license = { text = "MIT" }
+keywords = ["HDX", "OPHI"]
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3 :: Only",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Natural Language :: English",
+    "Operating System :: POSIX :: Linux",
+    "Operating System :: Unix",
+    "Operating System :: MacOS",
+    "Operating System :: Microsoft :: Windows",
+]
+requires-python = ">=3.8"
+
+dependencies = [
+    "hdx-python-api>= 6.3.4",
+    "hdx-python-country>= 3.8.1",
+    "hdx-python-scraper>= 2.5.0",
+    "hdx-python-utilities>= 3.7.4",
+]
+dynamic = ["version"]
+
+[project.readme]
+file = "README.md"
+content-type = "text/markdown"
+
+[project.urls]
+Homepage = "https://github.com/OCHA-DAP/hdx-scraper-ophi"
+
+[project.optional-dependencies]
+test = ["pytest", "pytest-check", "pytest-cov"]
+dev = ["pre-commit"]
+
+[project.scripts]
+run = "hdx.scraper.ophi.__main__:main"
+
+#########
+# Hatch #
+#########
+
+# Build
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/hdx"]
+
+[tool.hatch.build.hooks.vcs]
+version-file = "src/hdx/scraper/ophi/_version.py"
+
+[tool.hatch.metadata]
+allow-direct-references = true
+
+# Versioning
+
+[tool.hatch.version]
+source = "vcs"
+
+[tool.hatch.version.raw-options]
+local_scheme = "no-local-version"
+version_scheme = "python-simplified-semver"
+
+# Tests
+
+[tool.hatch.envs.hatch-test]
+features = ["test"]
+
+[[tool.hatch.envs.hatch-test.matrix]]
+python = ["3.12"]
+
+[tool.hatch.envs.hatch-test.scripts]
+run = """
+       pytest -c .config/pytest.ini --rootdir=. --junitxml=test-results.xml \
+       --cov --cov-config=.config/coveragerc --no-cov-on-fail \
+       --cov-report=lcov --cov-report=term-missing
+       """
+
+[tool.hatch.envs.hatch-static-analysis]
+dependencies = ["ruff==0.7.0"]
+
+[tool.hatch.envs.hatch-static-analysis.scripts]
+format-check = ["ruff format --config .config/ruff.toml --check --diff {args:.}",]
+format-fix = ["ruff format --config .config/ruff.toml {args:.}",]
+lint-check = ["ruff check --config .config/ruff.toml {args:.}",]
+lint-fix = ["ruff check --config .config/ruff.toml --fix {args:.}",]