[Initial Commit][l]: initial ckan aircan connector commit

.coveragerc

@@ -0,0 +1,5 @@
+[report]
+omit =
+    */site-packages/*
+    */python?.?/*
+    ckan/*

.gitignore

@@ -0,0 +1,42 @@
+.ropeproject
+node_modules
+bower_components
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+sdist/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.cache
+nosetests.xml
+coverage.xml
+
+# Sphinx documentation
+docs/_build/

.travis.yml

@@ -0,0 +1,13 @@
+language: python
+sudo: required
+python:
+    - "2.7"
+services:
+    - postgresql
+    - redis-server
+install:
+    - bash bin/travis-build.bash
+    - pip install coveralls
+script: sh bin/travis-run.sh
+after_success:
+    - coveralls

MANIFEST.in

@@ -0,0 +1,4 @@
+include README.rst
+include LICENSE
+include requirements.txt
+recursive-include ckanext/aircan_connector *.html *.json *.js *.less *.css *.mo

README.rst

@@ -0,0 +1,166 @@
+.. You should enable this project on travis-ci.org and coveralls.io to make
+   these badges work. The necessary Travis and Coverage config files have been
+   generated for you.
+
+.. image:: https://travis-ci.org/datopian/ckanext-aircan_connector.svg?branch=master
+    :target: https://travis-ci.org/datopian/ckanext-aircan_connector
+
+.. image:: https://coveralls.io/repos/datopian/ckanext-aircan_connector/badge.svg
+  :target: https://coveralls.io/r/datopian/ckanext-aircan_connector
+
+.. image:: https://pypip.in/download/ckanext-aircan_connector/badge.svg
+    :target: https://pypi.python.org/pypi//ckanext-aircan_connector/
+    :alt: Downloads
+
+.. image:: https://pypip.in/version/ckanext-aircan_connector/badge.svg
+    :target: https://pypi.python.org/pypi/ckanext-aircan_connector/
+    :alt: Latest Version
+
+.. image:: https://pypip.in/py_versions/ckanext-aircan_connector/badge.svg
+    :target: https://pypi.python.org/pypi/ckanext-aircan_connector/
+    :alt: Supported Python versions
+
+.. image:: https://pypip.in/status/ckanext-aircan_connector/badge.svg
+    :target: https://pypi.python.org/pypi/ckanext-aircan_connector/
+    :alt: Development Status
+
+.. image:: https://pypip.in/license/ckanext-aircan_connector/badge.svg
+    :target: https://pypi.python.org/pypi/ckanext-aircan_connector/
+    :alt: License
+
+=============
+ckanext-aircan_connector
+=============
+
+.. Put a description of your extension here:
+   What does it do? What features does it have?
+   Consider including some screenshots or embedding a video!
+
+
+------------
+Requirements
+------------
+
+For example, you might want to mention here which versions of CKAN this
+extension works with.
+
+
+------------
+Installation
+------------
+
+.. Add any additional install steps to the list below.
+   For example installing any non-Python dependencies or adding any required
+   config settings.
+
+To install ckanext-aircan_connector:
+
+1. Activate your CKAN virtual environment, for example::
+
+     . /usr/lib/ckan/default/bin/activate
+
+2. Install the ckanext-aircan_connector Python package into your virtual environment::
+
+     pip install ckanext-aircan_connector
+
+3. Add ``aircan_connector`` to the ``ckan.plugins`` setting in your CKAN
+   config file (by default the config file is located at
+   ``/etc/ckan/default/production.ini``).
+
+4. Restart CKAN. For example if you've deployed CKAN with Apache on Ubuntu::
+
+     sudo service apache2 reload
+
+
+---------------
+Config Settings
+---------------
+
+Document any optional config settings here. For example::
+
+    # The minimum number of hours to wait before re-checking a resource
+    # (optional, default: 24).
+    ckanext.aircan_connector.some_setting = some_default_value
+
+
+------------------------
+Development Installation
+------------------------
+
+To install ckanext-aircan_connector for development, activate your CKAN virtualenv and
+do::
+
+    git clone https://github.com/datopian/ckanext-aircan_connector.git
+    cd ckanext-aircan_connector
+    python setup.py develop
+    pip install -r dev-requirements.txt
+
+
+-----------------
+Running the Tests
+-----------------
+
+To run the tests, do::
+
+    nosetests --nologcapture --with-pylons=test.ini
+
+To run the tests and produce a coverage report, first make sure you have
+coverage installed in your virtualenv (``pip install coverage``) then run::
+
+    nosetests --nologcapture --with-pylons=test.ini --with-coverage --cover-package=ckanext.aircan_connector --cover-inclusive --cover-erase --cover-tests
+
+
+---------------------------------
+Registering ckanext-aircan_connector on PyPI
+---------------------------------
+
+ckanext-aircan_connector should be availabe on PyPI as
+https://pypi.python.org/pypi/ckanext-aircan_connector. If that link doesn't work, then
+you can register the project on PyPI for the first time by following these
+steps:
+
+1. Create a source distribution of the project::
+
+     python setup.py sdist
+
+2. Register the project::
+
+     python setup.py register
+
+3. Upload the source distribution to PyPI::
+
+     python setup.py sdist upload
+
+4. Tag the first release of the project on GitHub with the version number from
+   the ``setup.py`` file. For example if the version number in ``setup.py`` is
+   0.0.1 then do::
+
+       git tag 0.0.1
+       git push --tags
+
+
+----------------------------------------
+Releasing a New Version of ckanext-aircan_connector
+----------------------------------------
+
+ckanext-aircan_connector is availabe on PyPI as https://pypi.python.org/pypi/ckanext-aircan_connector.
+To publish a new version to PyPI follow these steps:
+
+1. Update the version number in the ``setup.py`` file.
+   See `PEP 440 <http://legacy.python.org/dev/peps/pep-0440/#public-version-identifiers>`_
+   for how to choose version numbers.
+
+2. Create a source distribution of the new version::
+
+     python setup.py sdist
+
+3. Upload the source distribution to PyPI::
+
+     python setup.py sdist upload
+
+4. Tag the new release of the project on GitHub with the version number from
+   the ``setup.py`` file. For example if the version number in ``setup.py`` is
+   0.0.2 then do::
+
+       git tag 0.0.2
+       git push --tags

bin/travis-build.bash

@@ -0,0 +1,43 @@
+#!/bin/bash
+set -e
+
+echo "This is travis-build.bash..."
+
+echo "Installing the packages that CKAN requires..."
+sudo apt-get update -qq
+sudo apt-get install solr-jetty
+
+echo "Installing CKAN and its Python dependencies..."
+git clone https://github.com/ckan/ckan
+cd ckan
+export latest_ckan_release_branch=`git branch --all | grep remotes/origin/release-v | sort -r | sed 's/remotes\/origin\///g' | head -n 1`
+echo "CKAN branch: $latest_ckan_release_branch"
+git checkout $latest_ckan_release_branch
+python setup.py develop
+pip install -r requirements.txt --allow-all-external
+pip install -r dev-requirements.txt --allow-all-external
+cd -
+
+echo "Creating the PostgreSQL user and database..."
+sudo -u postgres psql -c "CREATE USER ckan_default WITH PASSWORD 'pass';"
+sudo -u postgres psql -c 'CREATE DATABASE ckan_test WITH OWNER ckan_default;'
+
+echo "SOLR config..."
+# Solr is multicore for tests on ckan master, but it's easier to run tests on
+# Travis single-core. See https://github.com/ckan/ckan/issues/2972
+sed -i -e 's/solr_url.*/solr_url = http:\/\/127.0.0.1:8983\/solr/' ckan/test-core.ini
+
+echo "Initialising the database..."
+cd ckan
+paster db init -c test-core.ini
+cd -
+
+echo "Installing ckanext-aircan_connector and its requirements..."
+python setup.py develop
+pip install -r dev-requirements.txt
+
+echo "Moving test.ini into a subdir..."
+mkdir subdir
+mv test.ini subdir
+
+echo "travis-build.bash is done."

bin/travis-run.sh

@@ -0,0 +1,14 @@
+#!/bin/sh -e
+
+echo "NO_START=0\nJETTY_HOST=127.0.0.1\nJETTY_PORT=8983\nJAVA_HOME=$JAVA_HOME" | sudo tee /etc/default/jetty
+sudo cp ckan/ckan/config/solr/schema.xml /etc/solr/conf/schema.xml
+sudo service jetty restart
+
+nosetests --ckan \
+          --nologcapture \
+          --with-pylons=subdir/test.ini \
+          --with-coverage \
+          --cover-package=ckanext.aircan_connector \
+          --cover-inclusive \
+          --cover-erase \
+          --cover-tests

ckanext/__init__.py

@@ -0,0 +1,9 @@
+# encoding: utf-8
+
+# this is a namespace package
+try:
+    import pkg_resources
+    pkg_resources.declare_namespace(__name__)
+except ImportError:
+    import pkgutil
+    __path__ = pkgutil.extend_path(__path__, __name__)

ckanext/aircan_connector/action.py

@@ -0,0 +1,75 @@
+# encoding: utf-8
+
+import requests
+from ckan.common import config
+from ckan.plugins.toolkit import get_action
+#import urlparse
+import logging
+import json
+import tempfile
+
+log = logging.getLogger(__name__)
+
+def datapusher_submit(context, data_dict):
+    log.info("Submitting resource via API")
+    try:
+        res_id = data_dict['resource_id']
+        resource = get_resource_and_dataset(res_id)
+        resource_url = resource.get('url')
+        # fetch the resource data
+        log.info('Fetching from: {0}'.format(resource_url))
+        tmp_file = get_tmp_file(resource_url)
+        log.info('tmp_file.name: {0}'.format(tmp_file.name))
+        #records = read_from_file(tmp_file.name)
+        #log.info("records")
+        #log.info(records)
+        json_output_file_path = config['ckan.storage_path']+'/my.json'
+        log.info("json_output_file_path : {0}".format(json_output_file_path))
+        payload = {
+            "conf": {
+                "resource_i": res_id,
+                "schema_fields_array": ["id", "full_text", "FID"],
+                "csv_input": tmp_file.name,
+                "json_output": json_output_file_path
+            }
+        }
+        url = config['ckan.airflow.url']
+        log.info("Airflow URL: {0}".format(url))
+        response = requests.post(url,
+                                 data=payload,
+                                 headers={'Content-Type': 'application/json',
+                                          'Cache-Control': 'no-cache'}
+                                 )
+        #log.info(response)
+        response.raise_for_status()
+        log.info(response.json())
+        return response.json()
+    except Exception as e:
+        return {"success": False, "errors": [e]}
+
+    #url = 'http://ckan:8081/api/experimental/dags/ckan_api_load_multiple_steps/dag_runs'
+    #log.info("Aircan URL: " + url)
+    #response = requests.get(url)
+    ##response = requests.post(self.url, data=payload, headers=self.headers)
+    #response.raise_for_status()
+    #return response.json()
+
+
+def get_resource_and_dataset(resource_id):
+    """
+    Gets available information about the resource and its dataset from CKAN
+    """
+    res_dict = get_action('resource_show')(None, {'id': resource_id})
+    return res_dict
+
+def get_tmp_file(url):
+    filename = url.split('/')[-1].split('#')[0].split('?')[0]
+    log.info('get_tmp_file filename: {0}'.format(filename))
+    tmp_file = tempfile.NamedTemporaryFile(suffix=filename)
+    return tmp_file
+
+def read_from_file(filename):
+    content = None
+    with open(filename, 'r') as fd:
+        content = fd.read()
+    return content