Skip to content

Commit

Permalink
Mulltimodal pipeline improvement
Browse files Browse the repository at this point in the history 
- fixed the optimizer error in multimodal pipeline
- fixed the bug #564 'Example multi_modal_pipeline_genres failed'
- deleted the example of rating prediction
- optimized the process of NLP libraries import
- changed the data for multimodal example
- upgraded stemmer from Porter to Snowball
- fixed bug of merging multimodal data
- fixed bug of multimodal data shuffling while loading
- CNN now works on multioutput task
- Fixed the bug with incorrect type and shape of multioutput predictions
  • Loading branch information
@andreygetmanov
andreygetmanov committed Apr 1, 2022
1 parent a271006 commit 0c27d48
Show file tree
Hide file tree
Showing 12 changed files with 147 additions and 261 deletions.
34 changes: 21 additions & 13 deletions cases/multi_modal_genre_prediction.py
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import datetime
from examples.multi_modal_pipeline_genres import calculate_validation_metric, \
from examples.advanced.multi_modal_pipeline import calculate_validation_metric, \
generate_initial_pipeline_and_data, prepare_multi_modal_data
from fedot.core.composer.composer_builder import ComposerBuilder
from fedot.core.composer.gp_composer.gp_composer import PipelineComposerRequirements
Expand All @@ -10,16 +10,17 @@
from fedot.core.repository.tasks import Task, TaskTypesEnum


def run_multi_modal_case(files_path, is_visualise=True, timeout=datetime.timedelta(minutes=2)):
def run_multi_modal_case(files_path, is_visualise=True, timeout=datetime.timedelta(minutes=1)):
task = Task(TaskTypesEnum.classification)
images_size = (128, 128)

train_num, test_num, train_text, test_text = prepare_multi_modal_data(files_path, task,
images_size)
train_num, test_num, train_img, test_img, train_text, test_text = \
prepare_multi_modal_data(files_path, task, images_size)

pipeline, fit_data, predict_data = generate_initial_pipeline_and_data(images_size,
train_num, test_num,
train_text, test_text)
initial_pipeline, fit_data, predict_data = generate_initial_pipeline_and_data(images_size,
train_num, test_num,
train_img, test_img,
train_text, test_text)

# the search of the models provided by the framework that can be used as nodes in a pipeline for the selected task
available_model_types = get_operations_for_task(task=task, mode='model')
Expand All @@ -30,7 +31,7 @@ def run_multi_modal_case(files_path, is_visualise=True, timeout=datetime.timedel
composer_requirements = PipelineComposerRequirements(
primary=available_model_types,
secondary=available_model_types, max_arity=3,
max_depth=3, pop_size=5, num_of_generations=5,
max_depth=5, pop_size=5, num_of_generations=5,
crossover_prob=0.8, mutation_prob=0.8, timeout=timeout)

# GP optimiser parameters choice
Expand All @@ -43,14 +44,21 @@ def run_multi_modal_case(files_path, is_visualise=True, timeout=datetime.timedel
# the multi modal template (with data sources) is passed as initial assumption for composer
builder = ComposerBuilder(task=task).with_requirements(composer_requirements). \
with_metrics(metric_function).with_optimiser(parameters=optimiser_parameters).with_logger(logger=logger). \
with_initial_pipelines(pipeline).with_cache('multi_modal_opt.cache')
with_initial_pipelines([initial_pipeline]).with_cache('multi_modal_opt.cache')

pipeline.fit(input_data=fit_data)
# Create GP-based composer
composer = builder.build()

# the optimal pipeline generation by composition - the most time-consuming task
pipeline_evo_composed = composer.compose_pipeline(data=fit_data,
is_visualise=True)

pipeline_evo_composed.fit(input_data=fit_data)

if is_visualise:
pipeline.show()
pipeline_evo_composed.show()

prediction = pipeline.predict(predict_data, output_mode='labels')
prediction = pipeline_evo_composed.predict(predict_data, output_mode='labels')
err = calculate_validation_metric(test_text, prediction)

print(f'F1 micro for validation sample is {err}')
Expand All @@ -65,4 +73,4 @@ def download_mmdb_dataset():
if __name__ == '__main__':
download_mmdb_dataset()

run_multi_modal_case('cases/data/mmimdb', is_visualise=True)
run_multi_modal_case('cases/data/mm_imdb', is_visualise=True)
78 changes: 0 additions & 78 deletions cases/multi_modal_rating_prediction.py
View file

This file was deleted.

114 changes: 76 additions & 38 deletions examples/advanced/multi_modal_pipeline.py
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
import os

import numpy as np
from sklearn.metrics import roc_auc_score as roc_auc
from sklearn.metrics import f1_score as f1

from cases.dataset_preparation import unpack_archived_data
from fedot.core.pipelines.pipeline import Pipeline
Expand All @@ -14,75 +13,114 @@
from fedot.core.utils import fedot_project_root


def calculate_validation_metric(pred: OutputData, valid: InputData) -> float:
predicted = np.ravel(pred.predict)
real = np.ravel(valid.target)
def calculate_validation_metric(valid: InputData, pred: OutputData) -> float:
"""
Calculates F1 score for predicted data
err = roc_auc(y_true=real,
y_score=predicted)
:param valid: dataclass with true target
:param pred: dataclass with model's prediction
"""

real = valid.target
predicted = pred.predict

err = f1(y_true=real,
y_pred=predicted, average='micro')

return round(err, 2)


def prepare_multi_modal_data(files_path, task: Task, images_size=(128, 128), with_split=True):
path = os.path.join(str(fedot_project_root()), files_path)
"""
Imports data from 3 different sources (table, images and text)
unpack_archived_data(path)

data = InputData.from_json_files(path, fields_to_use=['votes', 'year'],
label='rating', task=task)
:param files_path: path to data
:param task: task to solve
:param images_size: the requested size in pixels, as a 2-tuple of (width, height)
:param with_split: if True, splits the sample on train/test
:return: 6 OutputData objects (2 with table data, 2 with images, 2 with text)
"""

class_labels = np.asarray([0 if t <= 7 else 1 for t in data.target])
data.target = class_labels
path = os.path.join(str(fedot_project_root()), files_path)
# unpacking of data archive
unpack_archived_data(path)
# import of table data
data_num = InputData.from_json_files(path, fields_to_use=['votes', 'rating'],
label='genres', task=task, is_multilabel=True, shuffle=False)

ratio = 0.5
class_labels = data_num.target
# train/test ratio
ratio = 0.6

img_files_path = f'{files_path}/*.jpeg'
img_path = os.path.join(str(fedot_project_root()), img_files_path)

# import of image data
data_img = InputData.from_image(images=img_path, labels=class_labels, task=task, target_size=images_size)

# import of text data
data_text = InputData.from_json_files(path, fields_to_use=['plot'],
label='rating', task=task,
data_type=DataTypesEnum.text)
data_text.target = class_labels
label='genres', task=task,
data_type=DataTypesEnum.text, is_multilabel=True, shuffle=False)

if with_split:
train_num, test_num = train_test_data_setup(data, shuffle_flag=False, split_ratio=ratio)
train_img, test_img = train_test_data_setup(data_img, shuffle_flag=False, split_ratio=ratio)
train_text, test_text = train_test_data_setup(data_text, shuffle_flag=False, split_ratio=ratio)

train_num, test_num = train_test_data_setup(data_num, shuffle_flag=True, split_ratio=ratio)
train_img, test_img = train_test_data_setup(data_img, shuffle_flag=True, split_ratio=ratio)
train_text, test_text = train_test_data_setup(data_text, shuffle_flag=True, split_ratio=ratio)
else:
train_num, test_num = data, data

train_num, test_num = data_num, data_num
train_img, test_img = data_img, data_img
train_text, test_text = data_text, data_text

return train_num, test_num, train_img, test_img, train_text, test_text


def generate_initial_pipeline_and_data(images_size,
train_num, test_num,
train_img, test_img,
train_text, test_text):
train_num, test_num,
train_img, test_img,
train_text, test_text):
"""
Generates initial pipeline for data from 3 different sources (table, images and text)
Each source is the primary node for its subpipeline
:param images_size: the requested size in pixels, as a 2-tuple of (width, height)
:param train_num: train sample of table data
:param test_num: test sample of table data
:param train_img: train sample of image data
:param test_img: test sample of image data
:param train_text: train sample of text data
:param test_text: test sample of text data
:return: pipeline object, 2 multimodal data objects (fit and predict)
"""

# image
ds_image = PrimaryNode('data_source_img/1')
image_node = SecondaryNode('cnn', nodes_from=[ds_image])
image_node.custom_params = {'image_shape': (images_size[0], images_size[1], 1),
'architecture': 'simplified',
'num_classes': 2,
'epochs': 15,
'batch_size': 128}
'num_classes': 5,
'epochs': 10,
'batch_size': 16,
'optimizer_parameters': {'loss': "binary_crossentropy",
'optimizer': "adam",
'metrics': 'categorical_crossentropy'}
}

# table
ds_table = PrimaryNode('data_source_table/2')
scaling_node = SecondaryNode('scaling', nodes_from=[ds_table])
numeric_node = SecondaryNode('rf', nodes_from=[scaling_node])
numeric_node = SecondaryNode('scaling', nodes_from=[ds_table])

# text
ds_text = PrimaryNode('data_source_text/3')
node_text_clean = SecondaryNode('text_clean', nodes_from=[ds_text])
text_node = SecondaryNode('tfidf', nodes_from=[node_text_clean])
text_node.custom_params = {'ngram_range': (1, 3), 'min_df': 0.001, 'max_df': 0.9}

pipeline = Pipeline(SecondaryNode('logit', nodes_from=[numeric_node, image_node, text_node]))
# combining all sources together
logit_node = SecondaryNode('logit', nodes_from=[image_node, numeric_node, text_node])
logit_node.custom_params = {'max_iter': 100000, 'random_state': 42}
pipeline = Pipeline(logit_node)

fit_data = MultiModalData({
'data_source_img/1': train_img,
Expand All @@ -106,20 +144,20 @@ def run_multi_modal_pipeline(files_path, is_visualise=False):
prepare_multi_modal_data(files_path, task, images_size)

pipeline, fit_data, predict_data = generate_initial_pipeline_and_data(images_size,
train_num, test_num,
train_img, test_img,
train_text, test_text)
train_num, test_num,
train_img, test_img,
train_text, test_text)

pipeline.fit(input_data=fit_data)

if is_visualise:
pipeline.show()

prediction = pipeline.predict(predict_data)
prediction = pipeline.predict(predict_data, output_mode='labels')

err = calculate_validation_metric(prediction, test_num)
err = calculate_validation_metric(test_text, prediction)

print(f'ROC AUC for validation sample is {err}')
print(f'F1 micro for validation sample is {err}')

return err

Expand Down
Binary file modified examples/data/multimodal.tar.gz
View file
Binary file not shown.
Loading

0 comments on commit 0c27d48

Please sign in to comment.