modify name of download_kenlm_models to include sentencepiece

Author: HugoLaurencon

ac_dc/README.md

@@ -21,7 +21,7 @@ Less importantly, you can also check how the anonymization and the normalization
 
 #### 2. Download everything you need
 
-To run the filtering code, it is necessary to download the dataset on which the filtering will take place, but also the necessary models, which are the Fasttext model for language identification (download [here](https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin)) and the Sentencepiece and KenLM models for tokenization and calculation of perplexity scores (download with the file [download_kenlm_models.py](https://github.com/bigscience-workshop/data_tooling/blob/master/ac_dc/download_kenlm_models.py)).
+To run the filtering code, it is necessary to download the dataset on which the filtering will take place, but also the necessary models, which are the Fasttext model for language identification (download [here](https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin)) and the Sentencepiece and KenLM models for tokenization and calculation of perplexity scores (download with the file [download_sentencepiece_kenlm_models.py](https://github.com/bigscience-workshop/data_tooling/blob/master/ac_dc/download_sentencepiece_kenlm_models.py)).
 
 #### 3. Choose the filtering parameters
 

ac_dc/download_kenlm_models.py …c/download_sentencepiece_kenlm_models.pyac_dc/download_kenlm_models.py renamed to ac_dc/download_sentencepiece_kenlm_models.py ac_dc/download_kenlm_models.py renamed to ac_dc/download_sentencepiece_kenlm_models.py

@@ -1,9 +1,9 @@
-"""Download kenlm models for supported languages (48) from Facebook.
+"""Download Sentencepiece and KenLM models for supported languages (48) from Facebook.
 
 Usage:
-    python download_kenlm_models.py --output_path /tmp/
+    python download_sentencepiece_kenlm_models.py --output_path /tmp/
 
-All kenlm language models will be saved under /tmp.
+All Sentencepiece and KenLM language models will be saved under /tmp.
 """
 
 import argparse
@@ -12,30 +12,35 @@
 from languages_id import langs_id
 
 
-def download_kenlm_models(output_path: str) -> None:
+def download_sentencepiece_kenlm_models(output_path: str) -> None:
+    supported_sentencepiece_langs = langs_id["sentencepiece_id"].dropna().unique()
+    for lang in supported_sentencepiece_langs:
+        try:
+            output_sentencepiece = subprocess.check_output(
+                f"wget http://dl.fbaipublicfiles.com/cc_net/lm/{lang}.sp.model -P {output_path}",
+                shell=True,
+            )
+        except:
+            print(f"Warning: Download failed for Sentencepiece model for language {lang}.")
+
     supported_kenlm_langs = langs_id["kenlm_id"].dropna().unique()
-    # unsupported_kenlm_langs = langs_id.loc[~langs_id.oscar_id.isin(langs_id.kenlm_id)].oscar_id.dropna().unique()
     for lang in supported_kenlm_langs:
         try:
             output_kenlm = subprocess.check_output(
                 f"wget http://dl.fbaipublicfiles.com/cc_net/lm/{lang}.arpa.bin -P {output_path}",
                 shell=True,
             )
-            output_sentencepiece = subprocess.check_output(
-                f"wget http://dl.fbaipublicfiles.com/cc_net/lm/{lang}.sp.model -P {output_path}",
-                shell=True,
-            )
         except:
             print(f"Warning: Download failed for KenLM model for language {lang}.")
 
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
-        description="Download kenlm models for supported languages."
+        description="Download Sentencepiece and KenLM models for supported languages."
     )
     parser.add_argument(
         "--output_path", type=str, default="/tmp/", help="Output path to save models."
     )
     args = parser.parse_args()
 
-    download_kenlm_models(output_path=args.output_path)
+    download_sentencepiece_kenlm_models(output_path=args.output_path)

ac_dc/filtering.py

@@ -2,9 +2,6 @@
 
 import fasttext
 
-# To download the fasttext model:
-# wget -O /tmp/lid.176.bin https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin
-
 import sentencepiece
 import kenlm