updates

dvc.yaml

@@ -1,4 +1,22 @@
 stages:
+  create_data:
+    cmd:  src/data/make_dataset.py
+    deps:
+      - src/data/make_dataset.py
+    outs:
+      - data/raw:
+          persist: true
+  process_data:
+    cmd: python src/data/process_data.py
+    deps:
+      - src/data/process_data.py
+    outs:
+      - data/processed/test.csv:
+          persist: true
+      - data/processed/train.csv:
+          persist: true
+      - data/processed/validation.csv:
+          persist: true
   train:
     cmd: python src/models/train_model.py
     deps:
@@ -20,14 +38,4 @@ stages:
     metrics:
       - reports/metrics.txt:
           cache: false
-  process_data:
-    cmd: python src/data/make_dataset.py
-    deps:
-      - src/data/make_dataset.py
-    outs:
-      - data/processed/test.csv:
-          persist: true
-      - data/processed/train.csv:
-          persist: true
-      - data/processed/validation.csv:
-          persist: true
+

src/data/process_data.py

@@ -1,8 +1,15 @@
 import pandas as pd
 
+
 def process_data(split='train'):
-    df= pd.DataFrame()
+    df = pd.DataFrame()
     dataset = pd.load_csv('summarization/data/raw/{}.csv'.format(split))
     df['article'] = dataset['article']
     df['highlights'] = dataset['highlights']
-    df.to_csv('summarization/data/processed/{}.csv'.format(split))
+    df.to_csv('summarization/data/processed/{}.csv'.format(split))
+
+
+if __name__ == '__name__':
+    process_data(split='train')
+    process_data(split='test')
+    process_data(split='validation')