Mulltimodal pipeline improvement

- fixed the optimizer error in multimodal pipeline
- fixed the bug #564 'Example multi_modal_pipeline_genres failed'
- deleted the example of rating prediction
- optimized the process of NLP libraries import
- changed the data for multimodal example
- upgraded stemmer from Porter to Snowball
- fixed bug of merging multimodal data
- fixed bug of multimodal data shuffling while loading

cases/multi_modal_genre_prediction.py

@@ -1,5 +1,5 @@
 import datetime
-from examples.multi_modal_pipeline_genres import calculate_validation_metric, \
+from examples.advanced.multi_modal_pipeline import calculate_validation_metric, \
     generate_initial_pipeline_and_data, prepare_multi_modal_data
 from fedot.core.composer.composer_builder import ComposerBuilder
 from fedot.core.composer.gp_composer.gp_composer import PipelineComposerRequirements
@@ -10,15 +10,15 @@
 from fedot.core.repository.tasks import Task, TaskTypesEnum
 
 
-def run_multi_modal_case(files_path, is_visualise=True, timeout=datetime.timedelta(minutes=2)):
+def run_multi_modal_case(files_path, is_visualise=True, timeout=datetime.timedelta(minutes=1)):
     task = Task(TaskTypesEnum.classification)
     images_size = (128, 128)
 
-    train_num, test_num, train_text, test_text = prepare_multi_modal_data(files_path, task,
-                                                                          images_size)
+    train_num, test_num, train_img, test_img, train_text, test_text = prepare_multi_modal_data(files_path, task, images_size)
 
     pipeline, fit_data, predict_data = generate_initial_pipeline_and_data(images_size,
                                                                           train_num, test_num,
+                                                                          train_img, test_img,
                                                                           train_text, test_text)
 
     # the search of the models provided by the framework that can be used as nodes in a pipeline for the selected task
@@ -30,7 +30,7 @@ def run_multi_modal_case(files_path, is_visualise=True, timeout=datetime.timedel
     composer_requirements = PipelineComposerRequirements(
         primary=available_model_types,
         secondary=available_model_types, max_arity=3,
-        max_depth=3, pop_size=5, num_of_generations=5,
+        max_depth=5, pop_size=5, num_of_generations=5,
         crossover_prob=0.8, mutation_prob=0.8, timeout=timeout)
 
     # GP optimiser parameters choice
@@ -43,14 +43,21 @@ def run_multi_modal_case(files_path, is_visualise=True, timeout=datetime.timedel
     # the multi modal template (with data sources) is passed as initial assumption for composer
     builder = ComposerBuilder(task=task).with_requirements(composer_requirements). \
         with_metrics(metric_function).with_optimiser(parameters=optimiser_parameters).with_logger(logger=logger). \
-        with_initial_pipelines(pipeline).with_cache('multi_modal_opt.cache')
+        with_initial_pipelines([pipeline]).with_cache('multi_modal_opt.cache')
 
-    pipeline.fit(input_data=fit_data)
+    # Create GP-based composer
+    composer = builder.build()
+
+    # the optimal pipeline generation by composition - the most time-consuming task
+    pipeline_evo_composed = composer.compose_pipeline(data=fit_data,
+                                                      is_visualise=True)
+
+    pipeline_evo_composed.fit(input_data=fit_data)
 
     if is_visualise:
         pipeline.show()
 
-    prediction = pipeline.predict(predict_data, output_mode='labels')
+    prediction = pipeline_evo_composed.predict(predict_data, output_mode='labels')
     err = calculate_validation_metric(test_text, prediction)
 
     print(f'F1 micro for validation sample is {err}')
@@ -65,4 +72,4 @@ def download_mmdb_dataset():
 if __name__ == '__main__':
     download_mmdb_dataset()
 
-    run_multi_modal_case('cases/data/mmimdb', is_visualise=True)
+    run_multi_modal_case('cases/data/mm_imdb', is_visualise=True)

examples/advanced/multi_modal_pipeline.py

@@ -1,7 +1,6 @@
 import os
 
-import numpy as np
-from sklearn.metrics import roc_auc_score as roc_auc
+from sklearn.metrics import f1_score as f1
 
 from cases.dataset_preparation import unpack_archived_data
 from fedot.core.pipelines.pipeline import Pipeline
@@ -14,55 +13,87 @@
 from fedot.core.utils import fedot_project_root
 
 
-def calculate_validation_metric(pred: OutputData, valid: InputData) -> float:
-    predicted = np.ravel(pred.predict)
-    real = np.ravel(valid.target)
+def calculate_validation_metric(valid: InputData, pred: OutputData) -> float:
+    """
+    Calculates F1 score for predicted data
 
-    err = roc_auc(y_true=real,
-                  y_score=predicted)
+    :param valid: true vector of predictions
+    :param pred: model's prediction
+    """
+
+    real = valid.target
+    predicted = pred.predict
+
+    err = f1(y_true=real,
+             y_pred=predicted, average='micro')
 
     return round(err, 2)
 
 
 def prepare_multi_modal_data(files_path, task: Task, images_size=(128, 128), with_split=True):
-    path = os.path.join(str(fedot_project_root()), files_path)
-
-    unpack_archived_data(path)
+    """
+    Imports data from 3 different sources (table, images and text)
 
-    data = InputData.from_json_files(path, fields_to_use=['votes', 'year'],
-                                     label='rating', task=task)
+    :param files_path: path to data
+    :param task: task to solve
+    :param images_size: the requested size in pixels, as a 2-tuple of (width, height)
+    :param with_split: if True, splits the sample on train/test
+    :return: 6 OutputData objects (2 with table data, 2 with images, 2 with text)
+    """
 
-    class_labels = np.asarray([0 if t <= 7 else 1 for t in data.target])
-    data.target = class_labels
+    path = os.path.join(str(fedot_project_root()), files_path)
+    # unpacking of data archive
+    unpack_archived_data(path)
+    # import of table data
+    data_num = InputData.from_json_files(path, fields_to_use=['votes', 'rating'],
+                                     label='genres', task=task, is_multilabel=True, shuffle=False)
 
-    ratio = 0.5
+    class_labels = data_num.target
+    # train/test ratio
+    ratio = 0.6
 
     img_files_path = f'{files_path}/*.jpeg'
     img_path = os.path.join(str(fedot_project_root()), img_files_path)
 
+    # import of image data
     data_img = InputData.from_image(images=img_path, labels=class_labels, task=task, target_size=images_size)
-
+    # import of text data
     data_text = InputData.from_json_files(path, fields_to_use=['plot'],
-                                          label='rating', task=task,
-                                          data_type=DataTypesEnum.text)
-    data_text.target = class_labels
+                                          label='genres', task=task,
+                                          data_type=DataTypesEnum.text, is_multilabel=True, shuffle=False)
 
     if with_split:
-        train_num, test_num = train_test_data_setup(data, shuffle_flag=False, split_ratio=ratio)
-        train_img, test_img = train_test_data_setup(data_img, shuffle_flag=False, split_ratio=ratio)
-        train_text, test_text = train_test_data_setup(data_text, shuffle_flag=False, split_ratio=ratio)
+
+        train_num, test_num = train_test_data_setup(data_num, shuffle_flag=True, split_ratio=ratio)
+        train_img, test_img = train_test_data_setup(data_img, shuffle_flag=True, split_ratio=ratio)
+        train_text, test_text = train_test_data_setup(data_text, shuffle_flag=True, split_ratio=ratio)
     else:
-        train_num, test_num = data, data
+
+        train_num, test_num = data_num, data_num
         train_img, test_img = data_img, data_img
         train_text, test_text = data_text, data_text
 
     return train_num, test_num, train_img, test_img, train_text, test_text
 
 
 def generate_initial_pipeline_and_data(images_size,
-                                    train_num, test_num,
-                                    train_img, test_img,
-                                    train_text, test_text):
+                                       train_num, test_num,
+                                       train_img, test_img,
+                                       train_text, test_text):
+    """
+    Generates initial pipeline for data from 3 different sources (table, images and text)
+
+    :param images_size: the requested size in pixels, as a 2-tuple of (width, height)
+    :param train_num: train sample of table data
+    :param test_num: test sample of table data
+    :param train_img: train sample of image data
+    :param test_img: test sample of image data
+    :param train_text: train sample of text data
+    :param test_text: test sample of text data
+    :return: pipeline object, 2 multimodal data objects (fit and predict)
+    """
+
+    # TODO make CNN to predict multilabel target
     # image
     ds_image = PrimaryNode('data_source_img/1')
     image_node = SecondaryNode('cnn', nodes_from=[ds_image])
@@ -74,15 +105,18 @@ def generate_initial_pipeline_and_data(images_size,
 
     # table
     ds_table = PrimaryNode('data_source_table/2')
-    scaling_node = SecondaryNode('scaling', nodes_from=[ds_table])
-    numeric_node = SecondaryNode('rf', nodes_from=[scaling_node])
+    numeric_node = SecondaryNode('scaling', nodes_from=[ds_table])
 
     # text
     ds_text = PrimaryNode('data_source_text/3')
     node_text_clean = SecondaryNode('text_clean', nodes_from=[ds_text])
     text_node = SecondaryNode('tfidf', nodes_from=[node_text_clean])
+    text_node.custom_params = {'ngram_range': (1, 3), 'min_df': 0.001, 'max_df': 0.9}
 
-    pipeline = Pipeline(SecondaryNode('logit', nodes_from=[numeric_node, image_node, text_node]))
+    # combining all sources together
+    logit_node = SecondaryNode('logit', nodes_from=[image_node, numeric_node, text_node])
+    logit_node.custom_params = {'max_iter': 100000, 'random_state': 42}
+    pipeline = Pipeline(logit_node)
 
     fit_data = MultiModalData({
         'data_source_img/1': train_img,
@@ -106,20 +140,20 @@ def run_multi_modal_pipeline(files_path, is_visualise=False):
         prepare_multi_modal_data(files_path, task, images_size)
 
     pipeline, fit_data, predict_data = generate_initial_pipeline_and_data(images_size,
-                                                                    train_num, test_num,
-                                                                    train_img, test_img,
-                                                                    train_text, test_text)
+                                                                          train_num, test_num,
+                                                                          train_img, test_img,
+                                                                          train_text, test_text)
 
     pipeline.fit(input_data=fit_data)
 
     if is_visualise:
         pipeline.show()
 
-    prediction = pipeline.predict(predict_data)
+    prediction = pipeline.predict(predict_data, output_mode='labels')
 
-    err = calculate_validation_metric(prediction, test_num)
+    err = calculate_validation_metric(test_text, prediction)
 
-    print(f'ROC AUC for validation sample is {err}')
+    print(f'F1 micro for validation sample is {err}')
 
     return err