import os import re import csv import json import urllib.request from collections import defaultdict, Counter COR_URL = "https://ordregister.dk/files/cor1.5.1.0.tsv" SOEGEHJAELP_URL = "https://ordregister.dk/files/corsoegehjaelp1.0.tsv" BASE_DIR = os.path.dirname(os.path.abspath(__file__)) DANISH_ALPHABET = "abcdefghijklmnopqrstuvwxyzæøå" DANISH_ORDER = {char: idx for idx, char in enumerate(DANISH_ALPHABET)} def danish_sort_key(word): if isinstance(word, (list, tuple)): word = word[0] return [DANISH_ORDER.get(char, ord(char)) for char in str(word)] def is_valid_word(w): return bool(re.match(r"^[a-zæøå]+$", w)) def save_simple_list(items, filepath_base, header="ord"): sorted_items = sorted(set(items), key=danish_sort_key) # TXT with open(f"{filepath_base}.txt", "w", encoding="utf-8") as f: for item in sorted_items: f.write(f"{item}\n") # JSON with open(f"{filepath_base}.json", "w", encoding="utf-8") as f: json.dump(sorted_items, f, ensure_ascii=False, indent=2) # CSV with open(f"{filepath_base}.csv", "w", encoding="utf-8", newline="") as f: writer = csv.writer(f) writer.writerow([header]) for item in sorted_items: writer.writerow([item]) print(f"Gemte {len(sorted_items)} elementer til {filepath_base}.[txt/json/csv]") def save_dict_rows(rows, filepath_base, fieldnames): # JSON with open(f"{filepath_base}.json", "w", encoding="utf-8") as f: json.dump(rows, f, ensure_ascii=False, indent=2) # CSV with open(f"{filepath_base}.csv", "w", encoding="utf-8", newline="") as f: writer = csv.DictWriter(f, fieldnames=fieldnames) writer.writeheader() writer.writerows(rows) print(f"Gemte {len(rows)} rækker til {filepath_base}.[json/csv]") def main(): print(f"1. Henter og læser COR-data...") req = urllib.request.Request(COR_URL, headers={"User-Agent": "OpenData-Extractor/1.0"}) all_valid_forms = set() pos_lemmas = defaultdict(set) pos_forms = defaultdict(set) # Grammatik strukturer verb_forms = defaultdict(dict) # lemma -> {form_type -> form} noun_genders = defaultdict(set) # lemma -> {fk, itk} noun_forms = defaultdict(set) # lemma -> {gram_tag} lemma_to_pos = defaultdict(set) # lemma -> {pos1, pos2} line_count = 0 with urllib.request.urlopen(req) as resp: for raw_line in resp: line_count += 1 if line_count % 150000 == 0: print(f"Læst {line_count} linjer...") line = raw_line.decode("utf-8").rstrip("\r\n") parts = line.split("\t") if len(parts) < 6: continue lemma = parts[1].strip() gram_desc = parts[3].strip() form = parts[4].strip().lower() normering = parts[5].strip() if normering not in ("N", "K"): continue lemma_clean = lemma.strip().lower() pos = gram_desc.split(".")[0] if gram_desc else "" if is_valid_word(form): all_valid_forms.add(form) if pos: pos_forms[pos].add(form) if is_valid_word(lemma_clean): if pos: pos_lemmas[pos].add(lemma_clean) lemma_to_pos[lemma_clean].add(pos) # Noun parsing if gram_desc.startswith("sb."): sub_parts = gram_desc.split(".") if len(sub_parts) >= 2 and sub_parts[1] in ("fk", "itk"): noun_genders[lemma_clean].add(sub_parts[1]) for sp in sub_parts: if sp in ("sg", "pl"): noun_forms[lemma_clean].add(sp) # Verb parsing if gram_desc.startswith("vb."): verb_forms[lemma_clean][gram_desc] = form print(f"Færdig med at læse COR. Starter generering af delmængder...") # ========================================== # A. GRAMMATIK-NØRDERIER # ========================================== gram_dir = os.path.join(BASE_DIR, "grammatik") # 1. Uregelmæssige verber irreg_verbs = [] for lem, forms in verb_forms.items(): praet = forms.get("vb.præt.akt") or forms.get("vb.præt.pass") perf = forms.get("vb.perf.part") inf = forms.get("vb.inf.akt") or lem if praet and not (praet.endswith("ede") or praet.endswith("te") or praet.endswith("edes") or praet.endswith("tes")): irreg_verbs.append({ "infinitiv": inf, "præsens": forms.get("vb.præs.akt", ""), "præteritum": praet, "perfektum_participium": perf or "", }) irreg_verbs.sort(key=lambda x: danish_sort_key(x["infinitiv"])) save_dict_rows(irreg_verbs, os.path.join(gram_dir, "uregelmaessige_verber"), ["infinitiv", "præsens", "præteritum", "perfektum_participium"]) # 2. Substantiver med valgfrit køn (fælleskøn og intetkøn) dual_genders = [lem for lem, genders in noun_genders.items() if {"fk", "itk"}.issubset(genders)] save_simple_list(dual_genders, os.path.join(gram_dir, "substantiver_valgfrit_koen"), header="substantiv") # 3. Plurale tantum (kun flertal) plurale_tantum = [lem for lem, f in noun_forms.items() if "pl" in f and "sg" not in f] save_simple_list(plurale_tantum, os.path.join(gram_dir, "plurale_tantum_kun_flertal"), header="substantiv") # 4. Singulare tantum (kun ental) singulare_tantum = [lem for lem, f in noun_forms.items() if "sg" in f and "pl" not in f] save_simple_list(singulare_tantum, os.path.join(gram_dir, "singulare_tantum_kun_ental"), header="substantiv") # 5. Ordklasse-kamæleoner (opslagsord der tilhører 2 eller flere ordklasser) kamaeleoner = [ {"lemma": lem, "antal_ordklasser": len(poses), "ordklasser": "|".join(sorted(poses))} for lem, poses in lemma_to_pos.items() if len(poses) >= 2 ] kamaeleoner.sort(key=lambda x: (-x["antal_ordklasser"], danish_sort_key(x["lemma"]))) save_dict_rows(kamaeleoner, os.path.join(gram_dir, "ordklasse_kamaeleoner"), ["lemma", "antal_ordklasser", "ordklasser"]) # ========================================== # B. ORDFÆNOMENER & KURIOSITETER # ========================================== kuri_dir = os.path.join(BASE_DIR, "kuriositeter") # 1. Semordnilaps (ord der stavet bagfra danner et andet gyldigt ord) semord_pairs = [] seen_pairs = set() for w in all_valid_forms: rev = w[::-1] if rev != w and len(w) >= 3 and rev in all_valid_forms: pair = tuple(sorted([w, rev], key=danish_sort_key)) if pair not in seen_pairs: seen_pairs.add(pair) semord_pairs.append({"ord_1": pair[0], "ord_2": pair[1], "laengde": len(pair[0])}) semord_pairs.sort(key=lambda x: (-x["laengde"], danish_sort_key(x["ord_1"]))) save_dict_rows(semord_pairs, os.path.join(kuri_dir, "semordnilaps_venderord"), ["ord_1", "ord_2", "laengde"]) # 2. Isogrammer (ord uden gentagne bogstaver, 10+ bogstaver) isograms = [w for w in all_valid_forms if len(w) >= 10 and len(set(w)) == len(w)] save_simple_list(isograms, os.path.join(kuri_dir, "isogrammer_uden_gentagne_bogstaver"), header="isogram") # 3. Konsonant-ekstremer (5 eller flere konsonanter i træk) vowels = set("aeiouyæøå") consonants = set("bcdfghjklmnpqrstvwxz") cons_5plus = [] for w in all_valid_forms: max_c = 0 cur_c = 0 for char in w: if char in consonants: cur_c += 1 if cur_c > max_c: max_c = cur_c else: cur_c = 0 if max_c >= 5: cons_5plus.append({"ord": w, "konsonanter_i_traek": max_c}) cons_5plus.sort(key=lambda x: (-x["konsonanter_i_traek"], danish_sort_key(x["ord"]))) save_dict_rows(cons_5plus, os.path.join(kuri_dir, "konsonanter_5_plus_i_traek"), ["ord", "konsonanter_i_traek"]) # 4. Vokal-ekstremer (3 eller flere vokaler i træk) vow_3plus = [] for w in all_valid_forms: max_v = 0 cur_v = 0 for char in w: if char in vowels: cur_v += 1 if cur_v > max_v: max_v = cur_v else: cur_v = 0 if max_v >= 3: vow_3plus.append({"ord": w, "vokaler_i_traek": max_v}) vow_3plus.sort(key=lambda x: (-x["vokaler_i_traek"], danish_sort_key(x["ord"]))) save_dict_rows(vow_3plus, os.path.join(kuri_dir, "vokaler_3_plus_i_traek"), ["ord", "vokaler_i_traek"]) # 5. Trippel-bogstaver og dobbelte dubletter tripple_chars = [w for w in all_valid_forms if any(w[i] == w[i+1] == w[i+2] for i in range(len(w)-2))] save_simple_list(tripple_chars, os.path.join(kuri_dir, "tre_ens_bogstaver_i_traek"), header="ord") consec_doubles = [w for w in all_valid_forms if any(w[i] == w[i+1] and w[i+2] == w[i+3] for i in range(len(w)-3))] save_simple_list(consec_doubles, os.path.join(kuri_dir, "paafolgende_dobbeltkonsonanter"), header="ord") # 6. Scrabble-guld: Korte ord med svære bogstaver (C, X, Z, Q, W på <= 4 bogstaver) svære_korte = [w for w in all_valid_forms if len(w) <= 4 and any(ch in w for ch in "cxzqw")] save_simple_list(svære_korte, os.path.join(kuri_dir, "scrabble_svaere_korte_ord"), header="ord") # 7. Q uden U q_no_u = [w for w in all_valid_forms if "q" in w and "qu" not in w] save_simple_list(q_no_u, os.path.join(kuri_dir, "q_uden_u"), header="ord") # ========================================== # C. TOTAL ORDKLASSEOPDELING & STATISTIK # ========================================== ordkl_dir = os.path.join(BASE_DIR, "ordklasser") # Primære ordklasse-lister pos_navne = { "sb": "substantiver_navneord", "vb": "verber_udsnagnsord", "adj": "adjektiver_tillaegsord", "adv": "adverbier_biord", "udråbsord": "udraabsord_interjektioner", "lydord": "lydord_onomatopoietika", "præp": "praepositioner_forholdsord", "konj": "konjunktioner_bindeord", "pron": "pronominer_stedord", "talord": "talord_numeralier", } pos_stats = [] for p_code, p_name in pos_navne.items(): lem_list = sorted(pos_lemmas.get(p_code, set()), key=danish_sort_key) form_list = sorted(pos_forms.get(p_code, set()), key=danish_sort_key) pos_stats.append({ "ordklasse_kode": p_code, "beskrivelse": p_name, "antal_grundformer": len(lem_list), "antal_boejede_former": len(form_list) }) if lem_list: save_simple_list(lem_list, os.path.join(ordkl_dir, f"{p_name}_grundformer"), header="lemma") # Statistik fil pos_stats.sort(key=lambda x: -x["antal_grundformer"]) save_dict_rows(pos_stats, os.path.join(ordkl_dir, "ordklasse_statistik"), ["ordklasse_kode", "beskrivelse", "antal_grundformer", "antal_boejede_former"]) # ========================================== # D. STAVEFEJL & FORVEKSLINGER (COR.SOEGEHJAELP) # ========================================== stave_dir = os.path.join(BASE_DIR, "stavefejl") print(f"2. Henter og behandler COR.SOEGEHJAELP...") sh_req = urllib.request.Request(SOEGEHJAELP_URL, headers={"User-Agent": "OpenData-Extractor/1.0"}) fejl_rows = [] with urllib.request.urlopen(sh_req) as resp: for raw_line in resp: line = raw_line.decode("utf-8").rstrip("\r\n") parts = line.split("\t") if len(parts) >= 4: officiel = parts[1].strip() gram_info = parts[2].strip() variant_fejl = parts[3].strip() fejl_rows.append({ "taste_stavefejl": variant_fejl, "korrekt_form": officiel, "ordklasse": gram_info }) fejl_rows.sort(key=lambda x: danish_sort_key(x["taste_stavefejl"])) save_dict_rows(fejl_rows, os.path.join(stave_dir, "typiske_stavefejl_og_rettelser"), ["taste_stavefejl", "korrekt_form", "ordklasse"]) print("Alle ekstra lister er genereret med succes!") if __name__ == "__main__": main()