Skip to content

Commit

Permalink
Integrated yq
Browse files Browse the repository at this point in the history
  • Loading branch information
fadado committed Aug 15, 2016
1 parent 81e9886 commit 3f2ae7b
Show file tree
Hide file tree
Showing 27 changed files with 1,775 additions and 69 deletions.
28 changes: 21 additions & 7 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -62,10 +62,11 @@ help:
# Tests
########################################################################

.PHONY: check cond expr loop macros syntax expand
.PHONY: check jqt format cond expr loop macros syntax expand csv yaml

check: expand jqt
check: expand jqt format
jqt: cond expr loop macros syntax
format: csv yaml

define TestJQT
# Run one example
Expand All @@ -83,6 +84,8 @@ $(1)-%: $(1)-%.jqt ;
$(1): $(sort $(subst tests/,,$(wildcard tests/$(1)-[0-9][0-9].jqt)))
endef

$(foreach t,cond expr loop macros syntax,$(eval $(call TestJQT,$(t))))

define TestGPP
# Run one example
$(2)-%.$(1):
Expand All @@ -96,10 +99,21 @@ $(2): $(sort $(subst tests/,,$(wildcard tests/$(2)-[0-9][0-9].$(1))))
endef

$(eval $(call TestGPP,jqt,expand))
$(eval $(call TestJQT,cond))
$(eval $(call TestJQT,expr))
$(eval $(call TestJQT,loop))
$(eval $(call TestJQT,macros))
$(eval $(call TestJQT,syntax))

define TestFileFormat
# Run one example
$(1)-%.sh:
echo "==> $$(basename $$@)"
$(SHELL) tests/$$@
# Run one example named without file suffix
$(1)-%: $(1)-%.sh ;
# Run all tests
$(1): $(sort $(subst tests/,,$(wildcard tests/$(1)-[0-9][0-9].sh)))
# Check output of all filters is empty for empty input
test -z "$$$$(for f in bin/$(1)2* bin/*2$(1); do echo | $$$$f; done)" \
|| { echo 1>&2 'EMPTY-FAILED'; false; }
endef

$(foreach t,csv yaml,$(eval $(call TestFileFormat,$(t))))

# vim:ai:sw=8:ts=8:noet:syntax=make
114 changes: 114 additions & 0 deletions bin/cq
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
#!/bin/bash

#
# Wraps jq to be able to process CSV
#

# Options
set -o errexit -o noglob -o nounset -o pipefail
shopt -s expand_aliases

# Script name
declare -r SELF=${0##*/}

# Format conversors
alias c2j=${0%$SELF}csv2json
alias j2c=${0%$SELF}json2csv
alias j2y=${0%$SELF}json2yaml

# Show help
function help {
cat <<EOF
${SELF} - commandline CSV processor
Usage: ${SELF} [options] <jq filter> [file]
${SELF} is a wrapper to jq for processing CSV input, applying the given
filter to it CSV text input and producing the filter's results as
CSV or JSON on standard output.
The options available are ${SELF} specific and also from jq. The ${SELF}
options are:
-h Show this help
-C Output using CSV format (default)
-J Output using JSON format
-Y Output using YAML format
-V Output the jq version
Some of the jq options include:
-e set the exit status code based on the output
-f Read filter from the file f
-s read (slurp) all inputs into an array; apply filter to it
-S sort keys of objects on output
--arg a v set variable \$a to value v
--argjson a v set variable \$a to JSON value v
--slurpfile a f set variable \$a to an array of values read from f
Not all jq options have sense using ${SELF}.
For more advanced filters see the jq(1) manpage and
https://stedolan.github.io/jq
EOF
exit $(( $# == 0 ))
}

# Entry point
function main {
local -i json=0 yaml=0

local opt
while getopts :hJKVY-: opt; do
case $opt in
h) help ;;
C) ;;
J) json=1 ;;
Y) yaml=1 ;;
V) exec jq --version ;;
-) case $OPTARG in
help) help ;;
CSV|csv) ;;
JSON|json) json=1 ;;
YAML|yaml) yaml=1 ;;
*) OPTIND=$((OPTIND-1))
break ;; # assume jq option
esac
;;
?) OPTIND=$((OPTIND-1))
break ;; # assume jq option
esac
done

shift $((OPTIND-1))
(( $# > 0 )) || help
[[ $1 == '--version' ]] && exec jq --version

# Is stdin a TTY?
if [[ -t 0 ]]; then
if (( $# >= 2 )); then
# Is the last parameter a regular file?
local last=${!#}
if [[ -f $last ]]; then
# reopen stdin
exec 0< "$last"
# remove last parameter
set -- "${@:1:$(($#-1))}"
fi
fi
fi

# Wrap jq
if (( json )); then
# Preserve JSON output
c2j | jq "$@"
elif (( yaml )); then
c2j | jq "$@" | j2y
else
c2j | jq "$@" | j2c
fi

# Exit status is that of the last command executed.
exit
}

# Call main
main "$@"

# vim:ai:sw=4:ts=4:et:syntax=sh
114 changes: 114 additions & 0 deletions bin/yq
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
#!/bin/bash

#
# Wraps jq to be able to process YAML
#

# Options
set -o errexit -o noglob -o nounset -o pipefail
shopt -s expand_aliases

# Script name
declare -r SELF=${0##*/}

# Format conversors
alias y2j=${0%$SELF}yaml2json
alias j2y=${0%$SELF}json2yaml
alias j2c=${0%$SELF}json2csv

# Show help
function help {
cat <<EOF
${SELF} - commandline YAML processor
Usage: ${SELF} [options] <jq filter> [file]
${SELF} is a wrapper to jq for processing YAML input, applying the given
filter to it YAML text input and producing the filter's results as
YAML or JSON on standard output.
The options available are ${SELF} specific and also from jq. The ${SELF}
options are:
-h Show this help
-J Output using JSON format
-C Output using CSV format
-Y Output using YAML format (default)
-V Output the jq version
Some of the jq options include:
-e set the exit status code based on the output
-f Read filter from the file f
-s read (slurp) all inputs into an array; apply filter to it
-S sort keys of objects on output
--arg a v set variable \$a to value v
--argjson a v set variable \$a to JSON value v
--slurpfile a f set variable \$a to an array of values read from f
Not all jq options have sense using ${SELF}.
For more advanced filters see the jq(1) manpage and
https://stedolan.github.io/jq
EOF
exit $(( $# == 0 ))
}

# Entry point
function main {
local -i json=0 csv=0

local opt
while getopts :hCJKVY-: opt; do
case $opt in
h) help ;;
C) csv=1 ;;
J) json=1 ;;
Y) ;;
V) exec jq --version ;;
-) case $OPTARG in
help) help ;;
CSV|csv) csv=1 ;;
JSON|json) json=1 ;;
YAML|yaml) ;;
*) OPTIND=$((OPTIND-1))
break ;; # assume jq option
esac
;;
?) OPTIND=$((OPTIND-1))
break ;; # assume jq option
esac
done

shift $((OPTIND-1))
(( $# > 0 )) || help
[[ $1 == '--version' ]] && exec jq --version

# Is stdin a TTY?
if [[ -t 0 ]]; then
if (( $# >= 2 )); then
# Is the last parameter a regular file?
local last=${!#}
if [[ -f $last ]]; then
# reopen stdin
exec 0< "$last"
# remove last parameter
set -- "${@:1:$(($#-1))}"
fi
fi
fi

# Wrap jq
if (( json )); then
# Preserve JSON output
y2j | jq "$@"
elif (( csv )); then
y2j | jq "$@" | j2c
else
y2j | jq "$@" | j2y
fi

# Exit status is that of the last command executed.
exit
}

# Call main
main "$@"

# vim:ai:sw=4:ts=4:et:syntax=sh
Loading

0 comments on commit 3f2ae7b

Please sign in to comment.