Mulltimodal pipeline improvement

- fixed the optimizer error in multimodal pipeline
- fixed the bug #564 'Example multi_modal_pipeline_genres failed'
- deleted the example of rating prediction
- optimized the process of NLP libraries import
- changed the data for multimodal example

cases/multi_modal_genre_prediction.py

@@ -1,5 +1,5 @@
 import datetime
-from examples.multi_modal_pipeline_genres import calculate_validation_metric, \
+from examples.advanced.multi_modal_pipeline import calculate_validation_metric, \
     generate_initial_pipeline_and_data, prepare_multi_modal_data
 from fedot.core.composer.composer_builder import ComposerBuilder
 from fedot.core.composer.gp_composer.gp_composer import PipelineComposerRequirements
@@ -10,15 +10,14 @@
 from fedot.core.repository.tasks import Task, TaskTypesEnum
 
 
-def run_multi_modal_case(files_path, is_visualise=True, timeout=datetime.timedelta(minutes=2)):
+def run_multi_modal_case(files_path, is_visualise=True, timeout=datetime.timedelta(minutes=1)):
     task = Task(TaskTypesEnum.classification)
     images_size = (128, 128)
 
-    train_num, test_num, train_text, test_text = prepare_multi_modal_data(files_path, task,
-                                                                          images_size)
+    train_img, test_img, train_text, test_text = prepare_multi_modal_data(files_path, task, images_size)
 
     pipeline, fit_data, predict_data = generate_initial_pipeline_and_data(images_size,
-                                                                          train_num, test_num,
+                                                                          train_img, test_img,
                                                                           train_text, test_text)
 
     # the search of the models provided by the framework that can be used as nodes in a pipeline for the selected task
@@ -30,7 +29,7 @@ def run_multi_modal_case(files_path, is_visualise=True, timeout=datetime.timedel
     composer_requirements = PipelineComposerRequirements(
         primary=available_model_types,
         secondary=available_model_types, max_arity=3,
-        max_depth=3, pop_size=5, num_of_generations=5,
+        max_depth=5, pop_size=5, num_of_generations=5,
         crossover_prob=0.8, mutation_prob=0.8, timeout=timeout)
 
     # GP optimiser parameters choice
@@ -43,14 +42,21 @@ def run_multi_modal_case(files_path, is_visualise=True, timeout=datetime.timedel
     # the multi modal template (with data sources) is passed as initial assumption for composer
     builder = ComposerBuilder(task=task).with_requirements(composer_requirements). \
         with_metrics(metric_function).with_optimiser(parameters=optimiser_parameters).with_logger(logger=logger). \
-        with_initial_pipelines(pipeline).with_cache('multi_modal_opt.cache')
+        with_initial_pipelines([pipeline]).with_cache('multi_modal_opt.cache')
 
-    pipeline.fit(input_data=fit_data)
+    # Create GP-based composer
+    composer = builder.build()
+
+    # the optimal pipeline generation by composition - the most time-consuming task
+    pipeline_evo_composed = composer.compose_pipeline(data=fit_data,
+                                                      is_visualise=True)
+
+    pipeline_evo_composed.fit(input_data=fit_data)
 
     if is_visualise:
         pipeline.show()
 
-    prediction = pipeline.predict(predict_data, output_mode='labels')
+    prediction = pipeline_evo_composed.predict(predict_data, output_mode='labels')
     err = calculate_validation_metric(test_text, prediction)
 
     print(f'F1 micro for validation sample is {err}')

examples/advanced/multi_modal_pipeline.py

@@ -1,7 +1,6 @@
 import os
 
-import numpy as np
-from sklearn.metrics import roc_auc_score as roc_auc
+from sklearn.metrics import f1_score as f1
 
 from cases.dataset_preparation import unpack_archived_data
 from fedot.core.pipelines.pipeline import Pipeline
@@ -14,12 +13,13 @@
 from fedot.core.utils import fedot_project_root
 
 
-def calculate_validation_metric(pred: OutputData, valid: InputData) -> float:
-    predicted = np.ravel(pred.predict)
-    real = np.ravel(valid.target)
+def calculate_validation_metric(valid: InputData, pred: OutputData) -> float:
 
-    err = roc_auc(y_true=real,
-                  y_score=predicted)
+    real = valid.target
+    predicted = pred.predict
+
+    err = f1(y_true=real,
+             y_pred=predicted, average='micro')
 
     return round(err, 2)
 
@@ -30,39 +30,42 @@ def prepare_multi_modal_data(files_path, task: Task, images_size=(128, 128), wit
     unpack_archived_data(path)
 
     data = InputData.from_json_files(path, fields_to_use=['votes', 'year'],
-                                     label='rating', task=task)
+                                     label='genres', task=task, is_multilabel=True)
 
-    class_labels = np.asarray([0 if t <= 7 else 1 for t in data.target])
-    data.target = class_labels
+    class_labels = data.target
 
-    ratio = 0.5
+    ratio = 0.6
 
     img_files_path = f'{files_path}/*.jpeg'
     img_path = os.path.join(str(fedot_project_root()), img_files_path)
 
+    # TODO fix shuffling of img data during import
+
     data_img = InputData.from_image(images=img_path, labels=class_labels, task=task, target_size=images_size)
 
     data_text = InputData.from_json_files(path, fields_to_use=['plot'],
-                                          label='rating', task=task,
-                                          data_type=DataTypesEnum.text)
-    data_text.target = class_labels
+                                          label='genres', task=task,
+                                          data_type=DataTypesEnum.text, is_multilabel=True)
 
     if with_split:
-        train_num, test_num = train_test_data_setup(data, shuffle_flag=False, split_ratio=ratio)
-        train_img, test_img = train_test_data_setup(data_img, shuffle_flag=False, split_ratio=ratio)
-        train_text, test_text = train_test_data_setup(data_text, shuffle_flag=False, split_ratio=ratio)
+
+        # train_num, test_num = train_test_data_setup(data, shuffle_flag=True, split_ratio=ratio)
+        train_img, test_img = train_test_data_setup(data_img, shuffle_flag=True, split_ratio=ratio)
+        train_text, test_text = train_test_data_setup(data_text, shuffle_flag=True, split_ratio=ratio)
     else:
-        train_num, test_num = data, data
+
+        # train_num, test_num = data, data
         train_img, test_img = data_img, data_img
         train_text, test_text = data_text, data_text
 
-    return train_num, test_num, train_img, test_img, train_text, test_text
+    return train_img, test_img, train_text, test_text
 
 
 def generate_initial_pipeline_and_data(images_size,
-                                    train_num, test_num,
-                                    train_img, test_img,
-                                    train_text, test_text):
+                                       # train_num, test_num,
+                                       train_img, test_img,
+                                       train_text, test_text):
+    # TODO make CNN to predict multilabel target
     # image
     ds_image = PrimaryNode('data_source_img/1')
     image_node = SecondaryNode('cnn', nodes_from=[ds_image])
@@ -72,26 +75,30 @@ def generate_initial_pipeline_and_data(images_size,
                                 'epochs': 15,
                                 'batch_size': 128}
 
+    # TODO fix bug with merging of table data
     # table
-    ds_table = PrimaryNode('data_source_table/2')
-    scaling_node = SecondaryNode('scaling', nodes_from=[ds_table])
-    numeric_node = SecondaryNode('rf', nodes_from=[scaling_node])
+    # ds_table = PrimaryNode('data_source_table/2')
+    # scaling_node = SecondaryNode('scaling', nodes_from=[ds_table])
+    # numeric_node = SecondaryNode('rf', nodes_from=[scaling_node])
 
     # text
     ds_text = PrimaryNode('data_source_text/3')
     node_text_clean = SecondaryNode('text_clean', nodes_from=[ds_text])
     text_node = SecondaryNode('tfidf', nodes_from=[node_text_clean])
+    text_node.custom_params = {'ngram_range': (1, 3), 'min_df': 0.001, 'max_df': 0.9}
 
-    pipeline = Pipeline(SecondaryNode('logit', nodes_from=[numeric_node, image_node, text_node]))
+    logit_node = SecondaryNode('logit', nodes_from=[text_node, image_node])
+    logit_node.custom_params = {'max_iter': 100000, 'random_state': 42}
+    pipeline = Pipeline(logit_node)
 
     fit_data = MultiModalData({
         'data_source_img/1': train_img,
-        'data_source_table/2': train_num,
+        # 'data_source_table/2': train_num,
         'data_source_text/3': train_text
     })
     predict_data = MultiModalData({
         'data_source_img/1': test_img,
-        'data_source_table/2': test_num,
+        # 'data_source_table/2': test_num,
         'data_source_text/3': test_text
     })
 
@@ -102,24 +109,24 @@ def run_multi_modal_pipeline(files_path, is_visualise=False):
     task = Task(TaskTypesEnum.classification)
     images_size = (128, 128)
 
-    train_num, test_num, train_img, test_img, train_text, test_text = \
+    train_img, test_img, train_text, test_text = \
         prepare_multi_modal_data(files_path, task, images_size)
 
     pipeline, fit_data, predict_data = generate_initial_pipeline_and_data(images_size,
-                                                                    train_num, test_num,
-                                                                    train_img, test_img,
-                                                                    train_text, test_text)
+                                                                          # train_num, test_num,
+                                                                          train_img, test_img,
+                                                                          train_text, test_text)
 
     pipeline.fit(input_data=fit_data)
 
     if is_visualise:
         pipeline.show()
 
-    prediction = pipeline.predict(predict_data)
+    prediction = pipeline.predict(predict_data, output_mode='labels')
 
-    err = calculate_validation_metric(prediction, test_num)
+    err = calculate_validation_metric(test_text, prediction)
 
-    print(f'ROC AUC for validation sample is {err}')
+    print(f'F1 micro for validation sample is {err}')
 
     return err