Notebook

In [1]:

"""https://en.wikipedia.org/wiki/Latin-1_Supplement_(Unicode_block)"""

import collections
import configparser
import pathlib
import re

GLOTTOLOG = pathlib.Path('~/Desktop/glottolog').expanduser()

REFERENCES = GLOTTOLOG / 'references'

CONFIG = REFERENCES / 'BIBFILES.ini'

BIBTEX = REFERENCES / 'bibtex'

HH_BIB = BIBTEX / 'hh.bib'

SKIP = {'degruyter.bib',
        'sil16.bib'}

ENCODING = 'utf-8'


class Config(configparser.ConfigParser):
    
    @classmethod
    def from_path(cls, path, *, encoding=ENCODING):
        inst = cls()
        with path.open(encoding=encoding) as f:
            inst.read_file(f)
        return inst


config = Config.from_path(CONFIG)
editable = {filename for filename, section in config.items()
            if 'enhancement of existing items possible' in section['curation'].lower()
            and filename not in SKIP}
editable

Out[1]:

{'anla.bib',
 'apics.bib',
 'asjp2010.bib',
 'autotyp.bib',
 'bahasa.bib',
 'bibliolux.bib',
 'bowern.bib',
 'eballiso2009.bib',
 'fabreall2009ann.bib',
 'gilbertese.bib',
 'gj.bib',
 'goba.bib',
 'guldemann.bib',
 'haspelmath.bib',
 'hedvig-tirailleur.bib',
 'lapolla-tibeto-burman.bib',
 'lewinmanx.bib',
 'ludger-paschen-germanic.bib',
 'marctang.bib',
 'mpieva.bib',
 'otomanguean.bib',
 'ozbib.bib',
 'phoible.bib',
 'sala.bib',
 'schikowski_chintang.bib',
 'seifart.bib',
 'silpng.bib',
 'sn.bib',
 'stampe.bib',
 'wals.bib',
 'weball.bib',
 'zorcpapers.bib',
 'zurich.bib'}

In [2]:

C1_CONTROL_CHAR = re.compile(r'[\u0080-\u009f]')


def count_control_chars(path, *, verbose=True, encoding=ENCODING):
    result = collections.Counter()
    with path.open(encoding=ENCODING) as f:
        for i, line in enumerate(f, start=1):
            chars = C1_CONTROL_CHAR.findall(line)
            if chars:
                codes = collections.Counter(f'U+{ord(c):04X}' for c in chars)
                stats = ', '.join(codepoint + ('*' * (count - 1) if count > 1 else '')
                                  for codepoint, count in codes.most_common())
                print(path.name, i, stats)
                result += codes
    return result


count_control_chars(HH_BIB)

Out[2]:

Counter()

In [3]:

bibfiles = sorted(BIBTEX.glob('*.bib'))
len(bibfiles)

Out[3]:

In [4]:

total = collections.Counter()

for path in bibfiles:
    total += count_control_chars(path)

benjamins.bib 28915 U+0097**, U+0092
benjamins.bib 28954 U+0093, U+0094
benjamins.bib 179283 U+0092
benjamins.bib 179331 U+0092**, U+0091*
benjamins.bib 179707 U+009A
benjamins.bib 182008 U+009A
benjamins.bib 201477 U+0093, U+0092, U+0094
benjamins.bib 201503 U+0092
benjamins.bib 201509 U+0092*, U+0093, U+0094
benjamins.bib 201535 U+0097*
benjamins.bib 211596 U+0092**
benjamins.bib 217734 U+0092
benjamins.bib 217785 U+0092
benjamins.bib 233323 U+009A*
benjamins.bib 233851 U+009A*
benjamins.bib 234367 U+009A
benjamins.bib 234637 U+009A
benjamins.bib 234647 U+009A***********
benjamins.bib 250920 U+0092
benjamins.bib 250930 U+0092**
benjamins.bib 250962 U+0096
benjamins.bib 251058 U+0092*
benjamins.bib 251092 U+0092*
benjamins.bib 251173 U+0092*, U+0091
benjamins.bib 251190 U+0096
benjamins.bib 251263 U+0092
benjamins.bib 251297 U+0092
benjamins.bib 268623 U+009A
benjamins.bib 275729 U+009A
benjamins.bib 310265 U+0092
benjamins.bib 310282 U+0097***, U+0091, U+0092
benjamins.bib 310299 U+0092, U+0093, U+0094
benjamins.bib 310315 U+0093*, U+0094*, U+0092*, U+0097*
benjamins.bib 310331 U+0092**, U+0093, U+0094
benjamins.bib 319306 U+0092
benjamins.bib 319316 U+0093*, U+0094*
benjamins.bib 319348 U+0093, U+0094
benjamins.bib 427487 U+0092
benjamins.bib 427536 U+0093**, U+0094**
benjamins.bib 427553 U+0093, U+0094
benjamins.bib 427570 U+0097*
benjamins.bib 442233 U+008A
benjamins.bib 442329 U+008A
benjamins.bib 469395 U+0097***
benjamins.bib 660000 U+0091, U+0092
benjamins.bib 660010 U+0091**, U+0092**
benjamins.bib 660027 U+0092***, U+0091**
benjamins.bib 660378 U+0092*******, U+0093, U+0094
degruyter.bib 530160 U+0091
degruyter.bib 568722 U+0096
degruyter.bib 568750 U+0091, U+0092
degruyter.bib 569352 U+0092*, U+0091
degruyter.bib 569366 U+0091, U+0092
degruyter.bib 569422 U+0091, U+0092
degruyter.bib 569576 U+0091, U+0092
degruyter.bib 569744 U+0091, U+0092
degruyter.bib 569758 U+0091, U+0092
degruyter.bib 569954 U+0091, U+0092
degruyter.bib 570122 U+0096
degruyter.bib 570752 U+0091, U+0092
degruyter.bib 578214 U+0096
degruyter.bib 578508 U+0096
degruyter.bib 578886 U+0096
degruyter.bib 579880 U+0091, U+0092
degruyter.bib 580650 U+0084, U+0094
degruyter.bib 580804 U+0096
degruyter.bib 581406 U+0092
degruyter.bib 581700 U+0091, U+0092
degruyter.bib 581770 U+0091, U+0092
degruyter.bib 581784 U+0091, U+0092
degruyter.bib 581882 U+0084*, U+0094*
degruyter.bib 582120 U+0091, U+0092
degruyter.bib 582302 U+0093, U+0094
degruyter.bib 582330 U+0084, U+0093
degruyter.bib 582344 U+0092, U+0084, U+0094
degruyter.bib 582414 U+0094, U+0093
degruyter.bib 582470 U+0084, U+0093
degruyter.bib 585229 U+0096
degruyter.bib 586629 U+0092*
degruyter.bib 622167 U+0092*
degruyter.bib 622257 U+0093, U+0094
degruyter.bib 622272 U+0092
degruyter.bib 622407 U+0092, U+0096
degruyter.bib 622422 U+0092
degruyter.bib 697231 U+0092*
degruyter.bib 697246 U+0093****, U+0094****, U+0092
degruyter.bib 697261 U+0092
degruyter.bib 697320 U+0092**
degruyter.bib 697350 U+0092*
degruyter.bib 697365 U+0092
degruyter.bib 713347 U+0096
degruyter.bib 713515 U+0096*
degruyter.bib 713809 U+0096
degruyter.bib 713977 U+0096*
degruyter.bib 715588 U+0096
degruyter.bib 715854 U+0096
degruyter.bib 716063 U+0086
degruyter.bib 716120 U+0084, U+0094
degruyter.bib 716470 U+0084, U+0094
degruyter.bib 716568 U+0084*, U+0094*
degruyter.bib 716582 U+0084, U+0094
degruyter.bib 716596 U+0096
degruyter.bib 716680 U+0096*
degruyter.bib 717660 U+0096
degruyter.bib 804023 U+0084, U+0093
degruyter.bib 804039 U+0084, U+0093, U+0096
degruyter.bib 804055 U+009A*, U+0084, U+0093
degruyter.bib 804104 U+0096
degruyter.bib 804152 U+0084, U+0093
degruyter.bib 804295 U+0084, U+0093
degruyter.bib 804311 U+0096
degruyter.bib 804327 U+0084, U+009E, U+0093
degruyter.bib 804343 U+0096
degruyter.bib 804359 U+0096
degruyter.bib 804375 U+0096
degruyter.bib 804390 U+0084, U+0093, U+0096
degruyter.bib 804447 U+0096*
degruyter.bib 804533 U+0084, U+0093, U+0096
degruyter.bib 804558 U+0096
degruyter.bib 804879 U+0096
degruyter.bib 805198 U+009A*
degruyter.bib 805246 U+0096
degruyter.bib 805321 U+0096
degruyter.bib 805336 U+0084, U+0093, U+0096
degruyter.bib 805361 U+0084**, U+0093*, U+0096, U+0094
degruyter.bib 805461 U+0093, U+0094
degruyter.bib 805485 U+0084, U+0093, U+0096
degruyter.bib 805558 U+0096***, U+009A*
degruyter.bib 805757 U+0096
degruyter.bib 805882 U+0096
degruyter.bib 805912 U+0084, U+0096, U+0093
degruyter.bib 805927 U+0096
degruyter.bib 805952 U+009A, U+0096
degruyter.bib 805984 U+0096
degruyter.bib 806166 U+0096**
degruyter.bib 806198 U+009A
degruyter.bib 806905 U+0096
degruyter.bib 808589 U+0096
degruyter.bib 808841 U+0096
degruyter.bib 810227 U+0096*
degruyter.bib 810997 U+0096
degruyter.bib 811053 U+0084, U+0094
degruyter.bib 845233 U+0093, U+0094
degruyter.bib 845250 U+0096***
degruyter.bib 845335 U+0096*
degruyter.bib 845480 U+0096*
degruyter.bib 845582 U+0093, U+0094
degruyter.bib 845604 U+0096
degruyter.bib 845715 U+0096
degruyter.bib 846030 U+0096
evobib.bib 3918 U+0094*
evobib.bib 5236 U+0092
evobib.bib 6018 U+0092
evobib.bib 7279 U+0094
evobib.bib 8854 U+0092*
evobib.bib 12428 U+0092
evobib.bib 12897 U+0092
evobib.bib 12980 U+0092
evobib.bib 13107 U+0094*
evobib.bib 13879 U+0088*
evobib.bib 16104 U+0094
evobib.bib 19572 U+0092
evobib.bib 20442 U+0094*
evobib.bib 21535 U+0094*
evobib.bib 22127 U+0094
evobib.bib 28276 U+0094
evobib.bib 29672 U+0092
evobib.bib 33284 U+0092
evobib.bib 33656 U+0094**
evobib.bib 34386 U+008C
evobib.bib 34387 U+008C
evobib.bib 34578 U+0092
evobib.bib 35756 U+0094
evobib.bib 37019 U+0094
evobib.bib 37080 U+0094*
evobib.bib 39648 U+0094
evobib.bib 40097 U+0092
evobib.bib 40353 U+0092
evobib.bib 40434 U+0092
ldh.bib 12171 U+0092
ldh.bib 12817 U+0093, U+0094
ldh.bib 13861 U+0096
sil16.bib 75677 U+0080
sil16.bib 85046 U+0080
sil16.bib 124097 U+0090
sil16.bib 129034 U+0080*
sil16.bib 137779 U+0090
sil16.bib 143544 U+0080**
sil16.bib 146284 U+0080*

In [5]:

total.most_common()

Out[5]:

[('U+0092', 101),
 ('U+0096', 70),
 ('U+0094', 58),
 ('U+0093', 42),
 ('U+009A', 30),
 ('U+0091', 26),
 ('U+0084', 26),
 ('U+0097', 17),
 ('U+0080', 9),
 ('U+008A', 2),
 ('U+0088', 2),
 ('U+008C', 2),
 ('U+0090', 2),
 ('U+0086', 1),
 ('U+009E', 1)]

In [6]:

for path in bibfiles:
    if path.name not in editable:
        continue

    old = path.read_text(encoding=ENCODING)
    new, replaced = C1_CONTROL_CHAR.subn('', old)

    if replaced:
        print(f'{path.name} replaced {replaced} characters')
        path.write_text(new, encoding=ENCODING)