1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
|
import pathlib
import re
import unicodedata
def safe_path(name):
simplified = ''.join([alnum_or_space(c) for c in unaccent(name.lower())])
return '-'.join(simplified.split())
def unaccent(s):
return ''.join(c for c in unicodedata.normalize('NFD', s) if unicodedata.category(c) != 'Mn')
def alnum_or_space(c):
if c.isalnum():
return c
else:
return ' '
def cleanup_text(s, lang):
s = re.sub('\'', '’', s)
s = re.sub(r'\.\.\.', '…', s)
s = re.sub(r'\. \. \.', '…', s)
s = cleanup_double_quotes(s, lang)
s = cleanup_paragraphs(s)
if lang == 'fr':
s = re.sub('“', '«', s)
s = re.sub('”', '»', s)
# Replace space by insecable spaces
s = re.sub(r' ([:?\!])', r' \1', s)
s = re.sub('« ', '« ', s)
s = re.sub(' »', ' »', s)
# Add missing insecable spaces
s = re.sub(r'([^ ]):', r'\1 :', s)
s = re.sub(r'([^ ])\?', r'\1 ?', s)
s = re.sub(r'([^ ])\!', r'\1 !', s)
s = re.sub(r'([^ ])»', r'\1 »', s)
s = re.sub(r'«([^ ])', r'« \1', s)
elif lang == 'en':
s = re.sub('« ', '“', s)
s = re.sub(' »', '”', s)
s = re.sub('«', '“', s)
s = re.sub('»', '”', s)
return s
def cleanup_double_quotes(s, lang):
res = ''
quoted = False
for c in s:
if c == '"':
if quoted:
quoted = False
if lang == 'fr':
res += '»'
elif lang == 'en':
res += '”'
else:
quoted = True
if lang == 'fr':
res += '«'
elif lang == 'en':
res += '“'
else:
res += c
return res
def cleanup_paragraphs(s):
ps = [f' {p.strip()}' for p in re.split(r'\n+', s) if p.strip()]
return '\n\n'.join(ps)
|
