#!/bin/bash
set -e  # abort on any errors

# Transform HTML metadata from source site into JSON

# for xpath 
XIDEL='xidel -s --input-format=html --output-format=json-wrapped'

# select all rows from the 2nd table element
XPATH_ENTRY='/html/body/table[2]/tbody/tr/td'

# with the xpath xidel returns a single JSON array with all table keys and values
# [["RN:","99002005","CU:","0  B2:30  1:55.149","NC:","11A","SR:","3400",... ]

# chunk the array into pairs of two and combine into a JSON object with key: value
#{
#  "RN:": "99002005",
#  "CU:": "0  B2:30  1:55.149",
#  "NC:": "11A",
#  "SR:": "3400",
#  "CS:": "3.388",
# ...
#} 
# The jq filter explained
# 1. assign the whole array to $row
# 2. create a range with a step of 2 over the lenght of the array, 0,2,4,...
# 3. create a object and use the range as index for the $row elements
# 3.5 remove right most colon from key
# 4. combine the list of objects into a single object with "add"
JQ_ARR2OBJ='[ .[] as $row | range(0; $row|length; 2) | {( $row[.] | rtrimstr(":")): ($row[.+1]) } ] | add'


while read RN
do
 $XIDEL --xpath "$XPATH_ENTRY" "raw/rn/metaData.cfm?RN=$RN" | jq "$JQ_ARR2OBJ" > "data/rn/$RN.json"
done < data/retrieval.numbers

# transform all sound records with jq, this is where the magic happens
jq -r -f transform.jq data/rn/*json > data/transformed.json