Pipeline updates

.dvc/config

@@ -0,0 +1,2 @@
+['remote "origin"']
+    url = https://dagshub.com/gagan3012/summarization.dvc

dvc.yaml

@@ -22,7 +22,6 @@ stages:
   process_data:
     cmd: python src/data/make_dataset.py
     deps:
-      - data/raw
       - src/data/make_dataset.py
     outs:
       - data/processed:

src/data/make_dataset.py

@@ -2,14 +2,16 @@ from datasets import load_dataset
 import pandas as pd
 
 
-def make_dataset(dataset='cnn_dailymail', split='train', version="3.0.0"):
+def make_dataset(dataset='cnn_dailymail', split='train'):
     """make dataset for summarisation"""
-    dataset = load_dataset(dataset, split=split, script_version=version)
+    dataset = load_dataset(dataset, '3.0.0', split=split)
     df = pd.DataFrame()
-    df['input_text'] = dataset['concepts']
-    df['output_text'] = dataset['target']
-    return df
+    df['input_text'] = dataset['article']
+    df['output_text'] = dataset['highlights']
+    df.to_csv('C:/Users/gbhat/Documents/GitHub/summarization/data/processed/{}.csv'.format(split, split))
 
 
 if __name__ == '__main__':
-    make_dataset(dataset='cnn_dailymail', split='train', version="3.0.0")
+    make_dataset(dataset='cnn_dailymail', split='train')
+    make_dataset(dataset='cnn_dailymail', split='test')
+    make_dataset(dataset='cnn_dailymail', split='validation')

src/models/model.py

@@ -303,9 +303,9 @@ class Summarization:
             tokenizer=self.tokenizer, model=self.model, output=outputdir
         )
 
-        # logger = MLFlowLogger(experiment_name="Summarization",tracking_uri="https://dagshub.com/gagan3012/summarization.mlflow")
+        MLlogger = MLFlowLogger(experiment_name="Summarization",tracking_uri="https://dagshub.com/gagan3012/summarization.mlflow")
 
-        logger = DAGsHubLogger()
+        logger = DAGsHubLogger(metrics_path='reports/metrics.txt')
 
         early_stop_callback = (
             [
@@ -324,7 +324,7 @@ class Summarization:
         gpus = 1 if use_gpu else 0
 
         trainer = Trainer(
-            logger=logger,
+            logger=[logger,MLlogger],
             callbacks=early_stop_callback,
             max_epochs=max_epochs,
             gpus=gpus,