fixing import issues

2026-09-02 23:27:31 +02:00
parent db55067708
commit df8df2729c
5 changed files with 107 additions and 3 deletions
--- a/prod_data_parse_scripts/parse_uc_osnova.sh
+++ b/prod_data_parse_scripts/parse_uc_osnova.sh
@@ -0,0 +1,104 @@
+#!/bin/sh
+
+set -eu
+
+if [ "$#" -lt 1 ] || [ "$#" -gt 2 ]; then
+    printf 'Usage: %s INPUT.csv [OUTPUT.csv]\n' "$0" >&2
+    exit 2
+fi
+
+input=$1
+output=${2:--}
+
+process_csv() {
+    awk -F ',' '
+    function clean_account(value, quoted) {
+        quoted = substr(value, 1, 1) == "\"" &&
+                 substr(value, length(value), 1) == "\""
+
+        if (quoted) {
+            value = substr(value, 2, length(value) - 2)
+        }
+
+        # Remove trailing non-numeric characters.
+        # Internal hyphens are preserved.
+        while (length(value) > 0 &&
+               substr(value, length(value), 1) !~ /[0-9]/) {
+            value = substr(value, 1, length(value) - 1)
+        }
+
+        if (quoted) {
+            return "\"" value "\""
+        }
+
+        return value
+    }
+
+    function key_value(value) {
+        if (substr(value, 1, 1) == "\"" &&
+            substr(value, length(value), 1) == "\"") {
+            return substr(value, 2, length(value) - 2)
+        }
+
+        return value
+    }
+
+    BEGIN {
+        OFS = ","
+    }
+
+    {
+        sub(/\r$/, "", $0)
+    }
+
+    NR == 1 {
+        printf "%s,\"deleted\"\n", $0
+        next
+    }
+
+    {
+        cleaned = clean_account($5)
+        key = key_value(cleaned)
+
+        $5 = cleaned
+
+        # Deleted:
+        # "000"  -> true
+        # "000/" -> cleaned to "000" -> true
+        # ""     -> true
+        #
+        # Everything else -> false
+        if (key == "000" || key == "") {
+            deleted = "\"true\""
+        } else {
+            deleted = "\"false\""
+        }
+
+        # Check duplicates only for non-deleted accounts
+        if (key != "000" && key != "") {
+            key_count[key]++
+
+            if (!(key in key_order)) {
+                key_order[key] = ++key_total
+            }
+        }
+
+        printf "%s,%s\n", $0, deleted
+    }
+
+    END {
+        for (key in key_order) {
+            if (key_count[key] > 1) {
+                printf "Non-unique column 5 key: %s (%d rows)\n",
+                       key, key_count[key] > "/dev/stderr"
+            }
+        }
+    }
+' "$input"
+}
+
+if [ "$output" = "-" ]; then
+    process_csv
+else
+    process_csv > "$output"
+fi