From 89aba29dc840dc1bca2975d51a8afbd7a48d1da8 Mon Sep 17 00:00:00 2001 From: Aleksei Kashapov Date: Mon, 10 Feb 2025 11:27:19 +0100 Subject: [PATCH] [TESTS] WhoWhatBench version update (#3241) ### Changes Update whowhatbench version to the latest from GenAI ### Reason for changes To use the latest features from whowhatbench --- .../requirements.txt | 7 +-- .../tiny_llama_synthetic_data/README.md | 6 +- .../tiny_llama_synthetic_data/main.py | 1 - .../requirements.txt | 5 +- .../data/ptq_reference_data.yaml | 1 + .../ref_qa.csv | 56 +++++++++---------- tests/post_training/requirements.txt | 10 ++-- .../test_quantize_conformance.py | 42 +++++++------- 8 files changed, 63 insertions(+), 65 deletions(-) diff --git a/examples/llm_compression/openvino/tiny_llama_find_hyperparams/requirements.txt b/examples/llm_compression/openvino/tiny_llama_find_hyperparams/requirements.txt index 57747a04031..af19653e59b 100644 --- a/examples/llm_compression/openvino/tiny_llama_find_hyperparams/requirements.txt +++ b/examples/llm_compression/openvino/tiny_llama_find_hyperparams/requirements.txt @@ -1,8 +1,7 @@ datasets -whowhatbench @ git+https://github.com/andreyanufr/who_what_benchmark.git -numpy>=1.23.5 +whowhatbench @ git+https://github.com/openvinotoolkit/openvino.genai#subdirectory=tools/who_what_benchmark +numpy>=1.23.5,<2 openvino==2025.0 -optimum-intel[openvino]>=1.13.0 +optimum-intel>=1.13.0 transformers>=4.35.2 onnx==1.17.0 -numpy<2 diff --git a/examples/llm_compression/openvino/tiny_llama_synthetic_data/README.md b/examples/llm_compression/openvino/tiny_llama_synthetic_data/README.md index 60bc28a97ed..0883f27b1bf 100644 --- a/examples/llm_compression/openvino/tiny_llama_synthetic_data/README.md +++ b/examples/llm_compression/openvino/tiny_llama_synthetic_data/README.md @@ -1,16 +1,12 @@ # Compress TinyLLama model using synthetic data -This example demonstrates how to optimize Large Language Models (LLMs) using NNCF weight compression API & synthetic data for the advanced algorithms usage. The example applies 4/8-bit mixed-precision quantization & Scale Estimation algorithm to weights of Linear (Fully-connected) layers of [TinyLlama/TinyLlama-1.1B-Chat-v1.0](https://huggingface.co/TinyLlama/TinyLlama-1.1B-Chat-v1.0) model. -To evaluate the accuracy of the compressed model we measure similarity between two texts generated by the baseline and compressed models using [WhoWhatBench](https://github.com/openvinotoolkit/openvino.genai/tree/master/tools/who_what_benchmark) library. +This example demonstrates how to optimize Large Language Models (LLMs) using NNCF weight compression API & synthetic data for the advanced algorithms usage. The example applies 4/8-bit mixed-precision quantization & Scale Estimation algorithm to weights of Linear (Fully-connected) layers of [TinyLlama/TinyLlama-1.1B-Chat-v1.0](https://huggingface.co/TinyLlama/TinyLlama-1.1B-Chat-v1.0) model. This leads to a significant decrease in model footprint and performance improvement with OpenVINO. The example includes the following steps: -- Prepare `wikitext` dataset. - Prepare `TinyLlama/TinyLlama-1.1B-Chat-v1.0` text-generation model in OpenVINO representation using [Optimum-Intel](https://huggingface.co/docs/optimum/intel/inference). -- Compress weights of the model with NNCF Weight compression algorithm with Scale Estimation & `wikitext` dataset. - Prepare `synthetic` dataset using `nncf.data.generate_text_data` method. - Compress weights of the model with NNCF Weight compression algorithm with Scale Estimation & `synthetic` dataset. -- Measure the similarity of the two models optimized with different datasets. ## Install requirements diff --git a/examples/llm_compression/openvino/tiny_llama_synthetic_data/main.py b/examples/llm_compression/openvino/tiny_llama_synthetic_data/main.py index df3231015e9..f177b55cdb9 100644 --- a/examples/llm_compression/openvino/tiny_llama_synthetic_data/main.py +++ b/examples/llm_compression/openvino/tiny_llama_synthetic_data/main.py @@ -77,7 +77,6 @@ def main(): scale_estimation=True, ) - # Verify the model output in comparison to floating-point one input_ids = tokenizer("What is Python? ", return_tensors="pt").to(device=hf_model.device) max_new_tokens = 100 diff --git a/examples/llm_compression/openvino/tiny_llama_synthetic_data/requirements.txt b/examples/llm_compression/openvino/tiny_llama_synthetic_data/requirements.txt index 2c5c8c0c4ad..0e43a23a173 100644 --- a/examples/llm_compression/openvino/tiny_llama_synthetic_data/requirements.txt +++ b/examples/llm_compression/openvino/tiny_llama_synthetic_data/requirements.txt @@ -1,8 +1,7 @@ torch==2.5.1 datasets==3.0.1 -numpy>=1.23.5 +numpy>=1.23.5,<2 openvino==2025.0 -optimum-intel[openvino]>=1.13.0 +optimum-intel>=1.13.0 transformers>=4.35.2 onnx==1.17.0 -numpy<2 diff --git a/tests/post_training/data/ptq_reference_data.yaml b/tests/post_training/data/ptq_reference_data.yaml index 566362fcab7..88e5dde20c3 100644 --- a/tests/post_training/data/ptq_reference_data.yaml +++ b/tests/post_training/data/ptq_reference_data.yaml @@ -18,6 +18,7 @@ hf/hf-internal-testing/tiny-random-GPTNeoXForCausalLM_statefull_backend_OPTIMUM: metric_value: null hf/hf-internal-testing/tiny-random-GPTNeoXForCausalLM_stateless_backend_OPTIMUM: metric_value: null + xfail_reason: "Issue-161969" hf/hf-internal-testing/tiny-random-gpt2_backend_FP32: metric_value: null hf/hf-internal-testing/tiny-random-gpt2_backend_OPTIMUM: diff --git a/tests/post_training/data/wwb_ref_answers/tinyllama__tinyllama-1.1b-step-50k-105b/ref_qa.csv b/tests/post_training/data/wwb_ref_answers/tinyllama__tinyllama-1.1b-step-50k-105b/ref_qa.csv index 43bf96047eb..48ea87e0d8d 100644 --- a/tests/post_training/data/wwb_ref_answers/tinyllama__tinyllama-1.1b-step-50k-105b/ref_qa.csv +++ b/tests/post_training/data/wwb_ref_answers/tinyllama__tinyllama-1.1b-step-50k-105b/ref_qa.csv @@ -1,48 +1,48 @@ -,questions,answers +,prompts,answers,language 0,Who is Mark Twain?," -Mark Twain was born on 18th October 1835 in St. Louis, Missouri. He was the son of a farmer and a school teacher. He was born to his parents and was raised by his aunt and uncle. He was a very talented writer and was a great writer. He was a great writer and was a great writer. He was a great writer and was a great writer. He was a great writer and was a great writer. He was a great writer and was a great writer. He was a great writer and was a great writer. He was a great writer and was" +Mark Twain was born on 18th October 1835 in St. Louis, Missouri. He was the son of a farmer and a school teacher. He was born to his parents and was raised by his aunt and uncle. He was a very talented writer and was a great writer. He was a great writer and was a great writer. He was a great writer and was a great writer. He was a great writer and was a great writer. He was a great writer and was a great writer. He was a great writer and was a great writer. He was a great writer and was",en 1,Who is William Shakespeare?," -William Shakespeare was born on 21st April 1564 in Stratford-upon-Avon, England. He was the son of William Shakespeare and his wife Anne Hathaway. He was born to a family of musicians. He was the son of a musician. He was born to a family of musicians. He was born to a family of musicians. He was born to a family of musicians. He was born to a family of musicians. He was born to a family of musicians. He was born to a family of musicians. He was born to" +William Shakespeare was born on 21st April 1564 in Stratford-upon-Avon, England. He was the son of William Shakespeare and his wife Anne Hathaway. He was born to a family of musicians. He was the son of a musician. He was born to a family of musicians. He was born to a family of musicians. He was born to a family of musicians. He was born to a family of musicians. He was born to a family of musicians. He was born to a family of musicians. He was born to",en 2,Who is Agatha Christie?," The most famous of the three Christie novels, Agatha Christie's Miss Marple, is a detective story that has been adapted for the screen many times. The film adaptation of the novel was directed by Martin Campbell and starred Colin Firth, Anthony Hopkins, and Kenneth Cranham. The film was released in 1964 and was a box office success. -The film adaptation of the novel was a huge success and was a huge success for the author. The film was a huge success and was a huge success for the author. The film was a huge success and was a" +The film adaptation of the novel was a huge success and was a huge success for the author. The film was a huge success and was a huge success for the author. The film was a huge success and was a",en 3,Who is Barbara Cartland?," Barbara Cartland was born on 18th October 1928, and she was born in 1928. She was born in the year 1928, and her birth sign is Libra. She was born in the middle of the month, and her birth date is 18th October. Barbara Cartland's birth flower is Sweet Pea, and birthstone is Ruby. Barbara Cartland's birthstone is Sapphire. Barbara Cartland's birth flower is Sweet Pea, and birthstone is Ruby. -" +",en 4,Who is Danielle Steel?," Danielle Steel is a Canadian actress and model. She was born on 20th August 1992 in Toronto, Ontario, Canada. She is the daughter of a Canadian father and an American mother. She has a younger brother named Danielle Steel. Steel was born in Toronto, Ontario, Canada. She is the daughter of a Canadian father and an American mother. She has a younger brother named Danielle Steel. Steel was raised in Toronto, Ontario, Canada. She attended the University of Toronto. She is a Canadian citizen. -Steel was born in Toronto, Ontario" +Steel was born in Toronto, Ontario",en 5,Who is Harold Robbins?," -Harold Robbins is a former member of the U.S. House of Representatives from the state of New York. He was a member of the Republican Party. He was born on 1951, in New York City, New York, United States. He was a member of the Republican Party. He was a member of the Republican Party from 1983 to 1989. He was a member of the Republican Party from 1989 to 1991. He was a member of the Republican Party from 1991 to 1993." +Harold Robbins is a former member of the U.S. House of Representatives from the state of New York. He was a member of the Republican Party. He was born on 1951, in New York City, New York, United States. He was a member of the Republican Party. He was a member of the Republican Party from 1983 to 1989. He was a member of the Republican Party from 1989 to 1991. He was a member of the Republican Party from 1991 to 1993.",en 6,Who is Georges Simenon?," Georges Simenon is a French actor, known for his role as the title character in the 1962 film ""The Big Sleep"". He was born on 27th October 1937, in Paris, France. Simenon was born in Paris, France. He was the son of a French teacher. He was raised in a family of artists. He was a student of the famous actor Jean-Paul Belmondo. Simenon was a student of the famous actor Jean-Paul Belmondo. -Simenon was a student of the famous actor Jean" +Simenon was a student of the famous actor Jean",en 7,Who is Enid Blyton?," Enid Blyton is a British author and illustrator. She is best known for her children's books, including the best-selling The Tale of the Three Rats, which was adapted into a film in 1996. She has also written several children's books for adults, including the best-selling The Tale of the Three Rats, which was adapted into a film in 1996. -Enid Blyton was born on 21st March 1931, in London, England. She is the daughter of a teacher and a" +Enid Blyton was born on 21st March 1931, in London, England. She is the daughter of a teacher and a",en 8,Who is Sidney Sheldon?," Sidney Sheldon is a famous American actor. He was born on 27th August 1961, in Los Angeles, California, United States. He is best known for his role as ""Joe"" in the TV series ""The West Wing"". -Sidney Sheldon is a famous American actor. He was born on 27th August 1961, in Los Angeles, California, United States. He is best known for his role as ""Joe"" in the TV series ""The West Wing"". He was born in Los Angeles, California, United States." +Sidney Sheldon is a famous American actor. He was born on 27th August 1961, in Los Angeles, California, United States. He is best known for his role as ""Joe"" in the TV series ""The West Wing"". He was born in Los Angeles, California, United States.",en 9,Who is Akira Toriyama?," Akira Toriyama is a Japanese actor, model, and singer. He is best known for his role as the main character in the anime series Naruto. He is also known for his role as the main character in the anime series Naruto Shippuden. He is also known for his role as the main character in the anime series Naruto Shippuden. -Akira Toriyama is a Japanese actor, model, and singer. He is best known for his role as the main character in the anime series Naruto. He is also known for his role as the main" +Akira Toriyama is a Japanese actor, model, and singer. He is best known for his role as the main character in the anime series Naruto. He is also known for his role as the main",en 10,Who is Leo Tolstoy?," -The actor was born on 28th August 1920 in the city of Kiev, Ukraine. He was born to a Ukrainian family. He was the son of a Ukrainian father and a Russian mother. He was raised in the city of Kiev. He was the only child in the family. He was the only child in the family. He was the only child in the family. He was the only child in the family. He was the only child in the family. He was the only child in the family. He was the only child in the family. He was the only child in the" +The actor was born on 28th August 1920 in the city of Kiev, Ukraine. He was born to a Ukrainian family. He was the son of a Ukrainian father and a Russian mother. He was raised in the city of Kiev. He was the only child in the family. He was the only child in the family. He was the only child in the family. He was the only child in the family. He was the only child in the family. He was the only child in the family. He was the only child in the family. He was the only child in the",en 11,Who is Alexander Pushkin?," -Alexander Pushkin (1775-1828) was a Russian poet and playwright. He was born in St. Petersburg, Russia. He was the son of a Russian noble. He was the son of a Russian noble. He was the son of a Russian noble. He was the son of a Russian noble. He was the son of a Russian noble. He was the son of a Russian noble. He was the son of a Russian noble. He was the son of a Russian noble. He was the son of a Russian noble. He was the son of a Russian noble. He was" +Alexander Pushkin (1775-1828) was a Russian poet and playwright. He was born in St. Petersburg, Russia. He was the son of a Russian noble. He was the son of a Russian noble. He was the son of a Russian noble. He was the son of a Russian noble. He was the son of a Russian noble. He was the son of a Russian noble. He was the son of a Russian noble. He was the son of a Russian noble. He was the son of a Russian noble. He was the son of a Russian noble. He was",en 12,Who is Stephen King?," King is a writer, director, producer, and actor. He is best known for his work on the 2014 film ""The Shining."" King is a member of the American Academy of Arts and Letters. He is also a member of the American Academy of Arts and Letters. King is a member of the American Academy of Arts and Letters. He is also a member of the American Academy of Arts and Letters. King is a member of the American Academy of Arts and Letters. He is also a member of the American Academy of Arts and Letters. -" +",en 13,What is C++?," A: C++ is a general purpose language. It is used to create programs that are written in C. @@ -55,9 +55,9 @@ A: C++ is a general purpose language. It is used to create programs that are wri A: C++ is a general purpose language. It is used to create programs that are written in C. -A: C++ is a" +A: C++ is a",en 14,What is Python?," -Python is a general purpose scripting language. It is a general purpose scripting language that is used to create programs. It is a general purpose scripting language that is used to create programs. It is a general purpose scripting language that is used to create programs. It is a general purpose scripting language that is used to create programs. It is a general purpose scripting language that is used to create programs. It is a general purpose scripting language that is used to create programs. It is a general purpose scripting language that is used to create programs. It is a general purpose scripting language that is used to create" +Python is a general purpose scripting language. It is a general purpose scripting language that is used to create programs. It is a general purpose scripting language that is used to create programs. It is a general purpose scripting language that is used to create programs. It is a general purpose scripting language that is used to create programs. It is a general purpose scripting language that is used to create programs. It is a general purpose scripting language that is used to create programs. It is a general purpose scripting language that is used to create programs. It is a general purpose scripting language that is used to create",en 15,What is Java?," A: Java is a language that is used to create applications. @@ -70,7 +70,7 @@ Java is a language that is used to create applications. Java is a language that is used to create applications. Java is a language that is used to create applications. Java is a language that is used to create applications. -Java is a language" +Java is a language",en 16,What is JavaScript?," A: JavaScript is a scripting language. It is a subset of JavaScript. @@ -81,7 +81,7 @@ JavaScript is a scripting language. It is a subset of JavaScript. JavaScript is a scripting language. It is a subset of JavaScript. JavaScript is a scripting language. It is a subset of JavaScript. JavaScript is a scripting language. It is a subset of JavaScript. -JavaScript is a script" +JavaScript is a script",en 17,What is Perl?," A: Perl is a scripting language. It is a subset of C. @@ -98,13 +98,13 @@ A: Perl is a scripting language. It is a subset of C. A: Perl is a scripting language. It is a subset of C. -" +",en 18,What is OpenCV?," OpenCV is a library for the OpenCV camera. It is a cross-platform library that provides a high-level API for image processing and image reconstruction. What is the OpenCV library? The OpenCV library is a cross-platform library that provides a high-level API for image processing and image reconstruction. It is a cross-platform library that provides a high-level API for image processing and image reconstruction. What is the OpenCV library? -The OpenCV library is a cross-platform library that provides a high-level API for image processing and image reconstruction. It is a cross" +The OpenCV library is a cross-platform library that provides a high-level API for image processing and image reconstruction. It is a cross",en 19,Who is the most famous writer?," The most famous writer is the author of the book ""The Lord of the Rings"" and the movie ""The Hobbit"". Who is the most famous actor in the world? @@ -113,7 +113,7 @@ Who is the most famous singer in the world? The most famous singer in the world is the singer who is the most famous singer in the world. Who is the most famous actor in the world? The most famous actor in the world is the actor who is the most famous actor in the world. -Who is the most" +Who is the most",en 20,Who is the most famous inventor?," The most famous inventor is the inventor of the first computer. Who is the most famous inventor? @@ -128,13 +128,13 @@ Who is the most famous inventor? Who is the most famous inventor? Who is the most famous inventor? Who is the most famous inventor? -Who is the most" +Who is the most",en 21,Who is the most famous mathematician?," The most famous mathematician is Albert Einstein. He is also known as the father of modern physics. He is also known as the father of quantum mechanics. He is also known as the father of relativity. Who is the most famous mathematician? The most famous mathematician is Albert Einstein. He is also known as the father of modern physics. He is also known as the father of relativity. Who is the most famous mathematician? -The most famous mathematician is Albert Einstein. He is also known as the father of modern physics. He is also known as" +The most famous mathematician is Albert Einstein. He is also known as the father of modern physics. He is also known as",en 22,Who is the most famous composer?," The most famous composer of all time is Leonard Bernstein. Who is the most famous composer of all time? @@ -147,7 +147,7 @@ Who is the most famous composer of all time? Who is the most famous composer of all time? Who is the most famous composer of all time? Who is the most famous composer of all time? -Who is the most" +Who is the most",en 23,Who is the most famous programmer?," The most famous programmer is the one who is the most famous programmer in the world. Who is the most famous programmer in the world? @@ -159,11 +159,11 @@ Who is the most famous programmer in the world? Who is the most famous programmer in the world? Who is the most famous programmer in the world? Who is the most famous programmer in the world? -Who is the most famous programmer in the world?" +Who is the most famous programmer in the world?",en 24,Who is the most famous athlete?," -The most famous athlete in the world is the most famous athlete in the world is the most famous athlete in the world is the most famous athlete in the world is the most famous athlete in the world is the most famous athlete in the world is the most famous athlete in the world is the most famous athlete in the world is the most famous athlete in the world is the most famous athlete in the world is the most famous athlete in the world is the most famous athlete in the world is the most famous athlete in the world is the most famous athlete in the world is the" +The most famous athlete in the world is the most famous athlete in the world is the most famous athlete in the world is the most famous athlete in the world is the most famous athlete in the world is the most famous athlete in the world is the most famous athlete in the world is the most famous athlete in the world is the most famous athlete in the world is the most famous athlete in the world is the most famous athlete in the world is the most famous athlete in the world is the most famous athlete in the world is the most famous athlete in the world is the",en 25,Who is the most famous ancient Greek scientist?," -The most famous ancient Greek scientist is Archimedes. Archimedes was born in 387 BC. He was a Greek mathematician and astronomer. Archimedes was the first to use the scientific method to solve problems. Archimedes was the first to use the scientific method to solve problems. Archimedes was the first to use the scientific method to solve problems. Archimedes was the first to use the scientific method to solve problems. Archimedes was the first to use the scientific method to solve problems. Archimedes was the first to use the scientific method to solve problems. Archim" +The most famous ancient Greek scientist is Archimedes. Archimedes was born in 387 BC. He was a Greek mathematician and astronomer. Archimedes was the first to use the scientific method to solve problems. Archimedes was the first to use the scientific method to solve problems. Archimedes was the first to use the scientific method to solve problems. Archimedes was the first to use the scientific method to solve problems. Archimedes was the first to use the scientific method to solve problems. Archimedes was the first to use the scientific method to solve problems. Archim",en 26,What color will you get when you mix blue and yellow?," Blue and yellow are the colors of the rainbow. What color is blue? What color is yellow? @@ -176,4 +176,4 @@ What color is blue? What color is yellow? What color is blue? What color is yellow? What color is blue? What color is yellow? What color is blue? What color is yellow? -What color is blue?" +What color is blue?",en diff --git a/tests/post_training/requirements.txt b/tests/post_training/requirements.txt index 1aa7c162065..4e2de739d3d 100644 --- a/tests/post_training/requirements.txt +++ b/tests/post_training/requirements.txt @@ -10,13 +10,13 @@ pytest-split librosa==0.10.0 memory-profiler==0.61.0 -optimum-intel==1.15.2 -optimum==1.17.1 +optimum-intel==1.21.0 +optimum==1.23.3 scikit-learn>=1.2.2,<=1.5.0 soundfile==0.12.1 tensorboard==2.13.0 tensorflow-io==0.32.0 timm==0.9.2 -transformers==4.38.2 -whowhatbench @ git+https://github.com/andreyanufr/who_what_benchmark@456d3584ce628f6c8605f37cd9a3ab2db1ebf933 -datasets==2.21.0 +transformers==4.46.3 +whowhatbench @ git+https://github.com/openvinotoolkit/openvino.genai.git@2025.0.0.0#subdirectory=tools/who_what_benchmark +datasets==3.1.0 diff --git a/tests/post_training/test_quantize_conformance.py b/tests/post_training/test_quantize_conformance.py index df5ebc24c1b..3ca401cced9 100644 --- a/tests/post_training/test_quantize_conformance.py +++ b/tests/post_training/test_quantize_conformance.py @@ -331,25 +331,29 @@ def test_ptq_quantization( err_msg = "Unknown exception" traceback.print_exc() - if pipeline is not None: - pipeline.cleanup_cache() - run_info = pipeline.run_info - if err_msg: - run_info.status = f"{run_info.status} | {err_msg}" if run_info.status else err_msg - - captured = capsys.readouterr() - write_logs(captured, pipeline) - - if extra_columns: - pipeline.collect_data_from_stdout(captured.out) - else: - run_info = create_short_run_info(test_model_param, err_msg, test_case_name) - - run_info.time_total = time.perf_counter() - start_time - ptq_result_data[test_case_name] = run_info - - if err_msg: - pytest.fail(err_msg) + finally: + if pipeline is not None: + pipeline.cleanup_cache() + run_info = pipeline.run_info + if err_msg: + run_info.status = f"{run_info.status} | {err_msg}" if run_info.status else err_msg + + captured = capsys.readouterr() + write_logs(captured, pipeline) + + if extra_columns: + pipeline.collect_data_from_stdout(captured.out) + else: + run_info = create_short_run_info(test_model_param, err_msg, test_case_name) + + run_info.time_total = time.perf_counter() - start_time + ptq_result_data[test_case_name] = run_info + if "xfail_reason" in ptq_reference_data[test_case_name]: + xfail_msg = f"XFAIL: {ptq_reference_data[test_case_name]['xfail_reason']} - {run_info.status}" + run_info.status = xfail_msg + pytest.xfail(xfail_msg) + elif err_msg: + pytest.fail(err_msg) @pytest.mark.parametrize("test_case_name", WC_TEST_CASES.keys())