split added

src/data/process_data.py

@@ -1,10 +1,18 @@
 import pandas as pd
+import yaml
+import os
 
 
 def process_data(split='train'):
 
+    with open("params.yml") as f:
+        params = yaml.safe_load(f)
+
     df = pd.read_csv('data/raw/{}.csv'.format(split))
     df.columns = ['Unnamed: 0', 'input_text', 'output_text']
+    df = df.sample(frac=params['split'], replace=True, random_state=1)
+    if os.path.exists("data/raw/{}.csv"):
+        os.remove("data/raw/{}.csv")
     df.to_csv('data/processed/{}.csv'.format(split))