Querying puppetdb for tags/variables

Hi All,
We’re using puppet (4, migrating to 5) and have all our facts in puppetdb. Has anyone used this for generating dashboard variables?
ie, generate a dashboard showing apache server-status for all nodes where (puppet role is webserver)

Alternatively I guess I have to see if I can provide a list of key=value pairs to influxdb via collectd

Hi,
We use a helper Script->
#!/bin/bash
usage () {
echo “Usage:”
cat <<-EOT
${0##*/} -o (csv|json|pretty) [ -l ] -v -f facts [ -s select ]
-o (csv|json|pretty) default is pretty for vt220
-e endpoint ( default = facts )
-l list
-s select virtual=true
-f fields

    ${0##*/} -e events -s failure 
    ${0##*/} -e events -s success 

    ${0##*/} -e nodes -s unreported

EOT
    exit 1

}

you must have read access to puppet certs

PUPPETCERT={PUPPETCERT:-"--cacert /var/lib/puppet/ssl/certs/ca.pem --cert /var/lib/puppet/ssl/certs/(hostname --fqdn).pem --key /var/lib/puppet/ssl/private_keys/$(hostname --fqdn).pem --tlsv1"}
PUPPETURL=“https://puppet.xx.xx:8081/v3
PUPPETCURLOPTS="-s --request GET $PUPPETCERT"
OUTFORMAT=pretty
VERBOSE=“false”
LISTFACTS=“false”

list_puppetdb_facts () {
curl $PUPPETCURLOPTS $PUPPETURL/fact-names | jq -r ‘.[]’ | grep -Ev “block|processor*|nsr|ipaddress_|network_|netmask_|mtu_|lvm_|macaddress_|hana_node_”
exit
}
while getopts “o:f:s:e:vlh” OPT; do
case $OPT in
o) OUTFORMAT=$OPTARG ;;
v) VERBOSE=“true” ;;
e) ENDPOINT=$OPTARG ;;
f) FACTSOPT=$OPTARG ;;
s) FACTSELECT=$OPTARG ;;
l) list_puppetdb_facts ;;
h) help ;;
esac
done

if test -z $1 ; then usage ; fi
if test -z $ENDPOINT ; then ENDPOINT=facts ; fi
if test $ENDPOINT == “reports” ; then
curl $PUPPETCURLOPTS $PUPPETURL/$ENDPOINT --data-urlencode ‘query=["=", “certname”, “XXX.x”]’
exit
fi

if test ENDPOINT == "events" ; then OFFSET=(( $( date +"%:z" | tr -d ‘0:’) * 3600 ))
#echo “2018-05-24T13:10:41.541Z” | jq -R ‘{datum: .|sub(".[0-9]+Z";“Z”)}’| jq ‘.datum|strptime("%Y-%m-%dT%H:%M:%SZ")|mktime + 7200 |todate’

/# curl $PUPPETCURLOPTS $PUPPETURL/$ENDPOINT --data-urlencode ‘query=["=", “certname”, “xxxxx.xx.x”]’
/# status -> success skipped failure noop

if test -z $FACTSELECT ; then FACTSELECT=“failure” ; fi

case $FACTSELECT in
failure|success|noop|skipped) echo -n “” ;;
*) echo “-s failure|success|noop|skipped”; usage ;;
esac

/# echo “FACTSELECT= " $FACTSELECT
curl $PUPPETCURLOPTS $PUPPETURL/$ENDPOINT --data-urlencode 'query=[”=", “latest-report?”, true]’ |
jq --argjson offset $OFFSET --arg status $FACTSELECT ’
.[]
|select ( .status == $status )
|select ( now - ( .“report-receive-time” | sub(".[0-9]+Z";“Z”) | strptime("%Y-%m-%dT%H:%M:%SZ")| mktime ) <= 7200 )
| { “report-receive-time”: .“report-receive-time” | sub(".[0-9]+Z";“Z”)| strptime("%Y-%m-%dT%H:%M:%SZ")| ( mktime + $offset ) | todate ,
“certname”: .“certname”,
“containing-class”: .“containing-class” ,
message: .message
}

exit
fi

if test $ENDPOINT == “nodes” ; then
if test -z $FACTSELECT ; then FACTSELECT=“unreported” ; fi
case $FACTSELECT in
unreported)
curl $PUPPETCURLOPTS $PUPPETURL/nodes | jq -r '.[]|select ( .report_timestamp == null ) |.name ’
;;
*) usage ;;
esac
exit
fi

FACTS=$(echo FACTSOPT | tr "," "\n") QUERY=( for fact in $FACTS ; do
jq -c -n --arg fact $fact ‘["=", “name”, $fact ]’
done | jq -s ‘if length > 1 then .|= [ “or” ] + . else .[] end’
)

/# virtual=vmware
/#‘select ( .virtual == “vmware” )’
/# echo {FACTSELECT%=*} {FACTSELECT#=}
if test ! -z "FACTSELECT" ; then JQSELECT="|select ( .{FACTSELECT%=
} == “${FACTSELECT#*=}” )"
fi

JQFILTER1=".[]|{certname: .certname , (.name): .value }"
JQFILTER2=“group_by(.certname)[] | add”${JQSELECT}

FACTSOUT=$( curl $PUPPETCURLOPTS $PUPPETURL/facts -d “query=$QUERY” | jq “$JQFILTER1” | jq -s “$JQFILTER2” )

/# convert commaseparated list to json array with leading dot
/# certname,virtual,hostname => [.certname , .virtual, .hostname]
FACTSARRAY=$( echo “certname,FACTSOPT" | sed -e "s/^/[.\"/g" -e "s/,/\", .\"/g" -e "s//”]/g" )
case $OUTFORMAT in
# csv) echo “$FACTSOUT” | jq -c . | json2csv -k certname,$FACTSOPT ;;
csv) echo “FACTSOUT" | jq -c -r "{FACTSARRAY}|@csv” ;;
json) echo “$FACTSOUT” | jq -c . ;;
pretty)
if type csvlook &>/dev/null && type header &>/dev/null
then echo “FACTSOUT" | jq -c -r "{FACTSARRAY}|@csv” | header -a hostname,$FACTSOPT| csvlook
else echo
echo
echo “please install packages csvkit and header for pretty output.
- csvkit https://github.com/wireservice/csvkit
- header https://github.com/jeroenjanssens/command-line-tools-for-data-science
or use -o json”
fi
;;
*) echo “unkown output format” ; usage ;;
esac
$VERBOSE && echo $QUERY
$VERBOSE && echo "curl $PUPPETCURLOPTS $PUPPETURL/facts -d ‘“query=$QUERY”’ "
$VERBOSE && echo “$JQFILTER1”
$VERBOSE && echo “$JQFILTER2”
$VERBOSE && echo “$JQSELECT”

Sample use
puppetdb.sh -f hostname,boardproductname,operatingsystem,manufacturer,operatingsystemrelease,macaddress,uptime_hours,nsr_pool,sysctl_kernel_sem_mns,physicalprocessorcount,virtual -o json|jq ‘(.hostname|tostring)+"," +if .boardproductname == null then “empty” else (.boardproductname|tostring) end +"," +(.operatingsystem|tostring) +"," +.operatingsystemrelease +","+if .manufacturer==“VMware, Inc.” then “Vmmware” else (.manufacturer|tostring) end +"," +(.macaddress|tostring)+","+if .nsr_pool==null then “-” else (.nsr_pool|tostring) end +"," +.physicalprocessorcount +"," +.virtual’|awk -F “,” ‘{ gsub (" “, “”, $0);print ““facter”,hostname=”$1"”"",boardproductname=""$2"""",operatingsystem=""$3"""",operatingsystemrelease=""$4"""",macaddress=""$6"""",virtual=""$9""",physicalprocessorcount="$8" value="$8""}’|xargs -i curl -s -0 -i -XPOST ‘http://localhost:8086/write?db=xx_puppet&precision=s’ --data-binary {}