add msra init data

Author: supercoderhawk

python/scripts/init_datasets.py

@@ -3,13 +3,17 @@
 from shutil import copyfile
 from dnlp.data_process.process_cws import ProcessCWS
 
+def init():
+  model_path = '../dnlp/models/'
+  if not os.path.exists(model_path):
+    os.makedirs(model_path)
 
 def copy():
   src_folder = '../../datasets/'
   dst_base_folder = '../dnlp/data/cws/'
   if not os.path.exists(dst_base_folder):
     os.makedirs(dst_base_folder)
-  files = ['pku_training.utf8','pku_test.utf8']
+  files = ['pku_training.utf8','pku_test.utf8','msr_training.utf8','msr_test.utf8']
   for f in files:
     copyfile(src_folder + f, dst_base_folder + f)
 
@@ -19,9 +23,13 @@ def build_cws_datasets():
   if not os.path.exists(base_folder):
     os.makedirs(base_folder)
   ProcessCWS(files=('pku_training.utf8',), base_folder=base_folder, name='pku_training')
-  dict_path = base_folder + 'pku_training_dict.utf8'
-  ProcessCWS(files=('pku_test.utf8',), dict_path=dict_path,base_folder=base_folder, name='pku_test',mode='test')
+  ProcessCWS(files=('msr_training.utf8',), base_folder=base_folder, name='msr_training')
+  pku_dict_path = base_folder + 'pku_training_dict.utf8'
+  ProcessCWS(files=('pku_test.utf8',), dict_path=pku_dict_path,base_folder=base_folder, name='pku_test',mode='test')
+  msr_dict_path = base_folder + 'msr_training_dict.utf8'
+  ProcessCWS(files=('msr_test.utf8',), dict_path=msr_dict_path, base_folder=base_folder, name='msr_test', mode='test')
 
 if __name__ == '__main__':
+  init()
   copy()
   build_cws_datasets()