#!/bin/bash
set -eo pipefail
# set -x

test -e "$(command -v xidel)" || (
  echo "ERR: Need xidel from https://www.videlibri.de/xidel.html"
  exit 1
)
test -e "$(command -v jq)" || (
  echo "ERR: Need jq from https://stedolan.github.io/jq/"
  exit 1
)

# Mapping of species id to common and scientific name

tail -n+56 data/species.map | jq -cR 'split("\t") as $row | {($row[0]): ($row[1])}' | jq -cs add >data/species.sci.names.json
head -n 55 data/species.map | jq -cR 'split("\t") as $row | {($row[0]): ($row[1])}' | jq -cs add >data/species.common.names.json

# Transform HTML metadata from source site into JSON

# for xpath
XIDEL='xidel -s --input-format=html --output-format=json-wrapped'

# select all rows from the 2nd table element
XPATH_ENTRY='/html/body/table[2]/tbody/tr/td'

# with the xpath xidel returns a single JSON array with all table keys and values
# [["RN:","99002005","CU:","0  B2:30  1:55.149","NC:","11A","SR:","3400",... ]

# chunk the array into pairs of two and combine into a JSON object with key: value
#{
#  "RN:": "99002005",
#  "CU:": "0  B2:30  1:55.149",
#  "NC:": "11A",
#  "SR:": "3400",
#  "CS:": "3.388",
# ...
#}
# The jq filter explained
# 1. assign the whole array to $row
# 2. create a range with a step of 2 over the lenght of the array, 0,2,4,...
# 3. create a object and use the range as index for the $row elements
# 3.5 remove right most colon from key
# 4. combine the list of objects into a single object with "add"

# shellcheck disable=SC2016
JQ_ARR2OBJ='[ .[] as $row | range(0; $row|length; 2) | {( $row[.] | rtrimstr(":")): ($row[.+1]) } ] | add'

test -d data/rn || mkdir -p data/rn

while read -r RN; do
  # input should exist
  test -f "raw/rn/metaData.cfm?RN=$RN" || continue
  # output should not exist
  test -f "data/rn/$RN.json" && continue
  $XIDEL --xpath "$XPATH_ENTRY" "raw/rn/metaData.cfm?RN=$RN" | jq -c "$JQ_ARR2OBJ" >"data/rn/$RN.json"
done <data/retrieval.numbers

# transform all records with jq, this is where the magic happens
./transform.jq data/rn/*json >data/transformed.json