Skip to content

Commit

Permalink
correct csv name (remove accents, space, an other non ASCII)
Browse files Browse the repository at this point in the history
  • Loading branch information
severus21 committed Mar 30, 2021
1 parent 99285df commit 7e3f252
Show file tree
Hide file tree
Showing 151 changed files with 57 additions and 1 deletion.
Binary file added __pycache__/__init__.cpython-38.pyc
Binary file not shown.
Binary file added __pycache__/__init__.cpython-39.pyc
Binary file not shown.
Binary file added __pycache__/jdl.cpython-38.pyc
Binary file not shown.
Binary file added __pycache__/jdl.cpython-39.pyc
Binary file not shown.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
30 changes: 29 additions & 1 deletion jdl.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
import random
import matplotlib.pyplot as plt
import math
import unicodedata
import re
from jdl.settings import BASE_DIR
from collections import Counter

Expand Down Expand Up @@ -52,7 +54,21 @@

# # Définition des fonctions de base


def slugify(value, allow_unicode=False):
"""
Taken from https://github.com/django/django/blob/master/django/utils/text.py
Convert to ASCII if 'allow_unicode' is False. Convert spaces or repeated
dashes to single dashes. Remove characters that aren't alphanumerics,
underscores, or hyphens. Convert to lowercase. Also strip leading and
trailing whitespace, dashes, and underscores.
"""
value = str(value)
if allow_unicode:
value = unicodedata.normalize('NFKC', value)
else:
value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore').decode('ascii')
value = re.sub(r'[^\w\s-]', '', value.lower())
return re.sub(r'[-\s]+', '-', value).strip('-_')


##### Fonction qui fait l'extraction de la base de région à partir de la base initiale
Expand Down Expand Up @@ -445,61 +461,73 @@ def absurde(x, y):

##### Fonctions d'extraction en fichiers csv
def export_pref(liste, name):
name = slugify(name)
df = pd.DataFrame(liste, columns=["prefixe"])
df.to_csv( os.path.join(DIR_PREF, f'{name}.csv'), index=False)
return

def import_pref(name):
name = slugify(name)
df = pd.read_csv(os.path.join(DIR_PREF, f'{name}.csv'), sep = ";")
lp = convliste(df, "prefixe")
return(lp)

def export_suf(liste, name):
name = slugify(name)
df = pd.DataFrame(liste, columns=["suffixe"])
df.to_csv(os.path.join(DIR_SUF, f'{name}.csv'), index=False)
return

def import_suf(name):
name = slugify(name)
df = pd.read_csv(os.path.join(DIR_SUF, f'{name}.csv'), sep = ";")
lp = convliste(df, "suffixe")
return(lp)

def export_pretiret(liste, name):
name = slugify(name)
df = pd.DataFrame(liste, columns=["pretiret"])
df.to_csv(os.path.join(DIR_PRETIRET, f'{name}.csv'), index=False)
return

def import_pretiret(name):
name = slugify(name)
df = pd.read_csv(os.path.join(DIR_PRETIRET, f'{name}.csv'), sep = ";")
lp = convliste(df, "pretiret")
return(lp)

def export_suftiret(liste, name):
name = slugify(name)
df = pd.DataFrame(liste, columns=["suftiret"])
df.to_csv(os.path.join(DIR_SUFTIRET, f'{name}.csv'), index=False)
return

def import_suftiret(name):
name = slugify(name)
df = pd.read_csv(os.path.join(DIR_SUFTIRET, f'{name}.csv'), sep = ";")
lp = convliste(df, "suftiret")
return(lp)

def export_miltiret(liste, name):
name = slugify(name)
df = pd.DataFrame(liste, columns=["miltiret"])
df.to_csv(os.path.join(DIR_MILTIRET, f'{name}.csv'), index=False)
return

def import_miltiret(name):
name = slugify(name)
df = pd.read_csv(os.path.join(DIR_MILTIRET, f'{name}.csv'), sep = ";")
lp = convliste(df, "miltiret")
return(lp)

def export_listordon(liste, name):
name = slugify(name)
df = pd.DataFrame(liste, columns=["liste"])
df.to_csv(os.path.join(DIR_LISTORDON, f'{name}.csv'), index=False)
return

def import_listordon(name):
name = slugify(name)
df = pd.read_csv(os.path.join(DIR_LISTORDON, f'{name}.csv'), sep = ";")
lp = convliste(df, "liste")
return(lp)
Expand Down
28 changes: 28 additions & 0 deletions slugify.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
#!/usr/bin/python3
import re
import os
import shutil
import unicodedata

def slugify(value, allow_unicode=False):
"""
Taken from https://github.com/django/django/blob/master/django/utils/text.py
Convert to ASCII if 'allow_unicode' is False. Convert spaces or repeated
dashes to single dashes. Remove characters that aren't alphanumerics,
underscores, or hyphens. Convert to lowercase. Also strip leading and
trailing whitespace, dashes, and underscores.
"""
value = str(value)
if allow_unicode:
value = unicodedata.normalize('NFKC', value)
else:
value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore').decode('ascii')
value = re.sub(r'[^\w\s-]', '', value.lower())
return re.sub(r'[-\s]+', '-', value).strip('-_')

if __name__ == '__main__':
for root, dirs, files in os.walk("csv", topdown=False):
for name in files:
path = os.path.join(root, name)
newpath = os.path.join(root, slugify(name[:-4])+'.csv' if name.endswith('.csv') else slugify(name))
shutil.move(path, newpath)

0 comments on commit 7e3f252

Please sign in to comment.