110 lines
2.2 KiB
Bash
110 lines
2.2 KiB
Bash
#!/bin/sh
|
|
|
|
set -eu
|
|
|
|
if [ "$#" -lt 1 ] || [ "$#" -gt 2 ]; then
|
|
printf 'Usage: %s INPUT.csv [OUTPUT.csv]\n' "$0" >&2
|
|
exit 2
|
|
fi
|
|
|
|
input=$1
|
|
output=${2:--}
|
|
|
|
process_csv() {
|
|
awk -F ',' '
|
|
function clean_account(value, quoted) {
|
|
quoted = substr(value, 1, 1) == "\"" &&
|
|
substr(value, length(value), 1) == "\""
|
|
|
|
if (quoted) {
|
|
value = substr(value, 2, length(value) - 2)
|
|
}
|
|
|
|
# Remove trailing non-numeric characters.
|
|
# Internal hyphens are preserved.
|
|
while (length(value) > 0 &&
|
|
substr(value, length(value), 1) !~ /[0-9]/) {
|
|
value = substr(value, 1, length(value) - 1)
|
|
}
|
|
|
|
if (quoted) {
|
|
return "\"" value "\""
|
|
}
|
|
|
|
return value
|
|
}
|
|
|
|
function key_value(value) {
|
|
if (substr(value, 1, 1) == "\"" &&
|
|
substr(value, length(value), 1) == "\"") {
|
|
return substr(value, 2, length(value) - 2)
|
|
}
|
|
|
|
return value
|
|
}
|
|
|
|
BEGIN {
|
|
OFS = ","
|
|
}
|
|
|
|
{
|
|
sub(/\r$/, "", $0)
|
|
}
|
|
|
|
NR == 1 {
|
|
printf "%s,\"deleted\"\n", $0
|
|
next
|
|
}
|
|
|
|
{
|
|
cleaned = clean_account($5)
|
|
key = key_value(cleaned)
|
|
|
|
if (key == "") {
|
|
cleaned = "\"000\""
|
|
key = "000"
|
|
}
|
|
|
|
$5 = cleaned
|
|
|
|
# Deleted:
|
|
# "000" -> true
|
|
# "000/" -> cleaned to "000" -> true
|
|
# "" -> changed to "000" -> true
|
|
#
|
|
# Everything else -> false
|
|
if (key == "000" || key == "") {
|
|
deleted = "\"true\""
|
|
} else {
|
|
deleted = "\"false\""
|
|
}
|
|
|
|
# Check duplicates only for non-deleted accounts
|
|
if (key != "000" && key != "") {
|
|
key_count[key]++
|
|
|
|
if (!(key in key_order)) {
|
|
key_order[key] = ++key_total
|
|
}
|
|
}
|
|
|
|
printf "%s,%s\n", $0, deleted
|
|
}
|
|
|
|
END {
|
|
for (key in key_order) {
|
|
if (key_count[key] > 1) {
|
|
printf "Non-unique column 5 key: %s (%d rows)\n",
|
|
key, key_count[key] > "/dev/stderr"
|
|
}
|
|
}
|
|
}
|
|
' "$input"
|
|
}
|
|
|
|
if [ "$output" = "-" ]; then
|
|
process_csv
|
|
else
|
|
process_csv > "$output"
|
|
fi
|