update

Author: Yi Zhang

defence/gramformer_gec.py

@@ -1,12 +1,3 @@
-'''
-The Gramformer project for GEC
-
-https://github.com/PrithivirajDamodaran/Gramformer
-
-pip install -U git+https://github.com/PrithivirajDamodaran/Gramformer.git
-python -m spacy download en_core_web_sm
-
-'''
 from gramformer import Gramformer
 import torch
 from tqdm import tqdm

generate_AE/attack_test.py

@@ -19,7 +19,7 @@
 from textattack.augmentation import Augmenter
 # transformation = CompositeTransformation([WordSwapRandomCharacterInsertion(),WordSwapRandomCharacterSubstitution(),WordSwapRandomCharacterDeletion(),WordSwapNeighboringCharacterSwap(),WordSwapQWERTY()])
 transformation = CompositeTransformation([WordSwapQWERTY()])
-constraints = [RepeatModification()]
+constraints = [RepeatModification(),StopwordModification()]
 
 import pandas as pd
 import clip

generate_AE/char_level.py

@@ -17,9 +17,9 @@
 from textattack.constraints.pre_transformation import RepeatModification
 from textattack.constraints.pre_transformation import StopwordModification
 from textattack.augmentation import Augmenter
-transformation = CompositeTransformation([WordSwapRandomCharacterInsertion(),WordSwapRandomCharacterSubstitution(),WordSwapRandomCharacterDeletion(),WordSwapNeighboringCharacterSwap(),WordSwapQWERTY()])
+# transformation = CompositeTransformation([WordSwapRandomCharacterInsertion(),WordSwapRandomCharacterSubstitution(),WordSwapRandomCharacterDeletion(),WordSwapNeighboringCharacterSwap(),WordSwapQWERTY()])
 # transformation = CompositeTransformation([WordSwapRandomCharacterSubstitution()])
-constraints = [RepeatModification()]
+constraints = [RepeatModification(), StopwordModification()]
 
 import pandas as pd
 import clip

version_defence.py

@@ -155,81 +155,80 @@ def defence_autocorrect(influent_sentence):
     start_time = time.time()
     origin_prompts = get_origin_prompt(origin_prompt_path) 
     for index, ori_prompt in origin_prompts.items():
-        if index == 19:
-            AEdata_path = f"./generate_AE/coco/char_AE/result_{index}.csv"
-            logger = setup_logger(f"adaptive_log/coco_update/spellchecker/log_char_{index}.log")
-            logger.info(f"sigma: {sigma}")
-            logger.info(f"num_inference_steps: {num_inference_steps}")
-            logger.info(f"num_batch: {num_batch}")
-            logger.info(f"batch_size: {batch_size}")
-            logger.info(AEdata_path)
-            logger.info(f"ori_prompt: {ori_prompt}")
-            df = pd.read_csv(AEdata_path)
-            ori_loss = []
-            for i in range(num_batch):
-                generator = torch.Generator(device).manual_seed(1023+i)
-                images = pipe([ori_prompt] * batch_size, num_inference_steps = num_inference_steps, generator = generator) 
-                for j in range(batch_size):
-                    ori_loss.append(calculate_text_image_distance(ori_prompt, images.images[j]))
-            logger.info(f"ori_loss: {len(ori_loss)} {ori_loss}")
-            logger.info(f"*" * 120)
-            for id in range(1, 2): 
-                efficient_n = 0
-                Non_AE, n = 0, 0
-                L_distance, AdvSt2i = [], []
-                robust_re, epsilon_re = [], []
-                sample_data = list(df[f"Column {id}"].dropna())
-                strings = [line.split(':')[0].strip() for line in sample_data[1:]]
-                logger.info(f"disturb rate: {id}")
-                logger.info(f"disturb_num: {sample_data[0]}")
-                n = 1
-                epsilon = 1000
-                for count in range(sample_num): 
-                    selected = random.choices(strings, k=1)[0]
-                    disturb_prompt = defence_spellchecker(selected)
-                    if disturb_prompt == ori_prompt:
-                        Non_AE += 1
-                        logger.info(f"dis_prompt: {selected}")
-                        logger.info(f"ori_prompt: {ori_prompt}")
-                        logger.info(f"same")
-                    else:
-                        logger.info(f"unsame")
-                        logger.info(f"selected: {selected}")
-                        logger.info(f"revised: {disturb_prompt}")
-                        L_distance.append(Levenshtein.distance(ori_prompt, disturb_prompt))
-                        whether_robust = cal_loss(ori_loss, disturb_prompt, ori_prompt)
-                        Non_AE += 1 if whether_robust else 0
-
-                    robust_left, robust_right, epsilon = calculate_R(Non_AE, n)
-                    robust_re.append((robust_left, robust_right))
-                    epsilon_re.append(epsilon)
-                    logger.info(f"stop_early: {efficient_n}")
-                    logger.info(f"Non_AE: {Non_AE}")
-                    logger.info(f"n: {n}")
-                    logger.info(f"robust reach: {robust_left} , {robust_right}")
-                    logger.info(f"epsilon reach: {epsilon}")
-                    print("*" * 120)
-                    logger.info(f"*" * 120)
-                    n += 1
+        AEdata_path = f"./generate_AE/coco/char_AE/result_{index}.csv"
+        logger = setup_logger(f"adaptive_log/coco_update/spellchecker/log_char_{index}.log")
+        logger.info(f"sigma: {sigma}")
+        logger.info(f"num_inference_steps: {num_inference_steps}")
+        logger.info(f"num_batch: {num_batch}")
+        logger.info(f"batch_size: {batch_size}")
+        logger.info(AEdata_path)
+        logger.info(f"ori_prompt: {ori_prompt}")
+        df = pd.read_csv(AEdata_path)
+        ori_loss = []
+        for i in range(num_batch):
+            generator = torch.Generator(device).manual_seed(1023+i)
+            images = pipe([ori_prompt] * batch_size, num_inference_steps = num_inference_steps, generator = generator) 
+            for j in range(batch_size):
+                ori_loss.append(calculate_text_image_distance(ori_prompt, images.images[j]))
+        logger.info(f"ori_loss: {len(ori_loss)} {ori_loss}")
+        logger.info(f"*" * 120)
+        for id in range(1, 2): 
+            efficient_n = 0
+            Non_AE, n = 0, 0
+            L_distance, AdvSt2i = [], []
+            robust_re, epsilon_re = [], []
+            sample_data = list(df[f"Column {id}"].dropna())
+            strings = [line.split(':')[0].strip() for line in sample_data[1:]]
+            logger.info(f"disturb rate: {id}")
+            logger.info(f"disturb_num: {sample_data[0]}")
+            n = 1
+            epsilon = 1000
+            for count in range(sample_num): 
+                selected = random.choices(strings, k=1)[0]
+                disturb_prompt = defence_spellchecker(selected)
+                if disturb_prompt == ori_prompt:
+                    Non_AE += 1
+                    logger.info(f"dis_prompt: {selected}")
+                    logger.info(f"ori_prompt: {ori_prompt}")
+                    logger.info(f"same")
+                else:
+                    logger.info(f"unsame")
+                    logger.info(f"selected: {selected}")
+                    logger.info(f"revised: {disturb_prompt}")
+                    L_distance.append(Levenshtein.distance(ori_prompt, disturb_prompt))
+                    whether_robust = cal_loss(ori_loss, disturb_prompt, ori_prompt)
+                    Non_AE += 1 if whether_robust else 0
+
+                robust_left, robust_right, epsilon = calculate_R(Non_AE, n)
+                robust_re.append((robust_left, robust_right))
+                epsilon_re.append(epsilon)
+                logger.info(f"stop_early: {efficient_n}")
+                logger.info(f"Non_AE: {Non_AE}")
+                logger.info(f"n: {n}")
+                logger.info(f"robust reach: {robust_left} , {robust_right}")
+                logger.info(f"epsilon reach: {epsilon}")
                 print("*" * 120)
                 logger.info(f"*" * 120)
-                logger.info(f"robust = {robust_re}")
-                logger.info(f"epsilon = {epsilon_re}")
-                logger.info(f"stop_early = {efficient_n}")
-                logger.info(f"E_n = {Non_AE}")
-                logger.info(f"n = {n}")
-                logger.info(f"AdvSt2i = {round(np.mean(AdvSt2i), 2)}")
-                logger.info(f"OriSt2i = {round(np.mean(ori_loss), 2)}")
-                logger.info(f"Levenshtein = {round(np.mean(L_distance), 2)}")
-                logger.info(f"robust = {robust_left} , {robust_right}")
-                logger.info(f"epsilon = {epsilon}")
-                
-                end_time = time.time()
-                elapsed_time = end_time - start_time
-                hours, remainder = divmod(elapsed_time, 3600)
-                minutes, seconds = divmod(remainder, 60)
-                print(f"time cost: {int(hours)} hours, {int(minutes)} minutes, {int(seconds)} seconds")
-                logger.info(f"time cost: {int(hours)} hours, {int(minutes)} minutes, {int(seconds)} seconds")
-                logger.info(f"&" * 150)
+                n += 1
+            print("*" * 120)
+            logger.info(f"*" * 120)
+            logger.info(f"robust = {robust_re}")
+            logger.info(f"epsilon = {epsilon_re}")
+            logger.info(f"stop_early = {efficient_n}")
+            logger.info(f"E_n = {Non_AE}")
+            logger.info(f"n = {n}")
+            logger.info(f"AdvSt2i = {round(np.mean(AdvSt2i), 2)}")
+            logger.info(f"OriSt2i = {round(np.mean(ori_loss), 2)}")
+            logger.info(f"Levenshtein = {round(np.mean(L_distance), 2)}")
+            logger.info(f"robust = {robust_left} , {robust_right}")
+            logger.info(f"epsilon = {epsilon}")
+            
+            end_time = time.time()
+            elapsed_time = end_time - start_time
+            hours, remainder = divmod(elapsed_time, 3600)
+            minutes, seconds = divmod(remainder, 60)
+            print(f"time cost: {int(hours)} hours, {int(minutes)} minutes, {int(seconds)} seconds")
+            logger.info(f"time cost: {int(hours)} hours, {int(minutes)} minutes, {int(seconds)} seconds")
+            logger.info(f"&" * 150)
 
 

version_sample.py

@@ -135,74 +135,73 @@ def get_origin_prompt(origin_prompt_path):
     robust_left, robust_right = 0, 0
     origin_prompts = get_origin_prompt(origin_prompt_path) 
     for index, ori_prompt in origin_prompts.items():
-        if index >= 7 and index < 31:
-            efficient_m, efficient_n = 0, 0
-            AEdata_path = f"./generate_AE/coco/char_AE/result_{index}.csv"
-            logger = setup_logger(f"adaptive_log/log_char_{index}.log")
-            logger.info(f"sigma: {sigma}")
-            logger.info(f"num_inference_steps: {num_inference_steps}")
-            logger.info(f"num_batch: {num_batch}")
-            logger.info(f"batch_size: {batch_size}")
-            logger.info(AEdata_path)
-            logger.info(f"ori_prompt: {ori_prompt}")
-            df = pd.read_csv(AEdata_path)
-            ori_loss = []
-            for i in range(num_batch):
-                generator = torch.Generator(device).manual_seed(1023+i)
-                images = pipe([ori_prompt] * batch_size, num_inference_steps = num_inference_steps, generator = generator)
-                for j in range(batch_size):
-                    ori_loss.append(calculate_text_image_distance(ori_prompt, images.images[j]))
-            logger.info(f"ori_loss: {len(ori_loss)} {ori_loss}")
-            logger.info(f"*" * 120)
-            for id in range(2, 3): 
-                efficient_n = 0
-                Non_AE, n = 0, 0
-                L_distance, AdvSt2i = [], []
-                robust_re, epsilon_re = [], []
-                sample_data = list(df[f"Column {id}"].dropna())
-                strings = [line.split(':')[0].strip() for line in sample_data[1:]]
-                logger.info(f"disturb rate: {id}")
-                logger.info(f"disturb_num: {sample_data[0]}")
-                n = 1
-                epsilon = 1000
-                for count in range(sample_num):
-                    disturb_prompt = random.choices(strings, k=1)[0]
-                    L_distance.append(Levenshtein.distance(ori_prompt, disturb_prompt))
-                    whether_robust = cal_loss(ori_loss, disturb_prompt, ori_prompt)
-                    Non_AE += 1 if whether_robust else 0
-                    robust_left, robust_right, epsilon = calculate_R(Non_AE, n)
-                    robust_re.append((robust_left, robust_right))
-                    epsilon_re.append(epsilon)
-                    logger.info(f"stop_early: {efficient_n}")
-                    logger.info(f"futility: {efficient_m}")
-                    logger.info(f"Non_AE: {Non_AE}")
-                    logger.info(f"n: {n}")
-                    logger.info(f"robust reach: {robust_left} , {robust_right}")
-                    logger.info(f"epsilon reach: {epsilon}")
-                    print("*" * 120)
-                    logger.info(f"*" * 120)
-                    n += 1
+        efficient_m, efficient_n = 0, 0
+        AEdata_path = f"./generate_AE/coco/char_AE/result_{index}.csv"
+        logger = setup_logger(f"adaptive_log/log_char_{index}.log")
+        logger.info(f"sigma: {sigma}")
+        logger.info(f"num_inference_steps: {num_inference_steps}")
+        logger.info(f"num_batch: {num_batch}")
+        logger.info(f"batch_size: {batch_size}")
+        logger.info(AEdata_path)
+        logger.info(f"ori_prompt: {ori_prompt}")
+        df = pd.read_csv(AEdata_path)
+        ori_loss = []
+        for i in range(num_batch):
+            generator = torch.Generator(device).manual_seed(1023+i)
+            images = pipe([ori_prompt] * batch_size, num_inference_steps = num_inference_steps, generator = generator)
+            for j in range(batch_size):
+                ori_loss.append(calculate_text_image_distance(ori_prompt, images.images[j]))
+        logger.info(f"ori_loss: {len(ori_loss)} {ori_loss}")
+        logger.info(f"*" * 120)
+        for id in range(2, 3): 
+            efficient_n = 0
+            Non_AE, n = 0, 0
+            L_distance, AdvSt2i = [], []
+            robust_re, epsilon_re = [], []
+            sample_data = list(df[f"Column {id}"].dropna())
+            strings = [line.split(':')[0].strip() for line in sample_data[1:]]
+            logger.info(f"disturb rate: {id}")
+            logger.info(f"disturb_num: {sample_data[0]}")
+            n = 1
+            epsilon = 1000
+            for count in range(sample_num):
+                disturb_prompt = random.choices(strings, k=1)[0]
+                L_distance.append(Levenshtein.distance(ori_prompt, disturb_prompt))
+                whether_robust = cal_loss(ori_loss, disturb_prompt, ori_prompt)
+                Non_AE += 1 if whether_robust else 0
+                robust_left, robust_right, epsilon = calculate_R(Non_AE, n)
+                robust_re.append((robust_left, robust_right))
+                epsilon_re.append(epsilon)
+                logger.info(f"stop_early: {efficient_n}")
+                logger.info(f"futility: {efficient_m}")
+                logger.info(f"Non_AE: {Non_AE}")
+                logger.info(f"n: {n}")
+                logger.info(f"robust reach: {robust_left} , {robust_right}")
+                logger.info(f"epsilon reach: {epsilon}")
                 print("*" * 120)
                 logger.info(f"*" * 120)
-                logger.info(f"robust = {robust_re}")
-                logger.info(f"epsilon = {epsilon_re}")
-                logger.info(f"stop_early = {efficient_n}")
-                logger.info(f"futility = {efficient_m}")
-                logger.info(f"Non_AE = {Non_AE}")
-                logger.info(f"n = {n}")
-                logger.info(f"AdvSt2i = {round(np.mean(AdvSt2i), 2)}")
-                logger.info(f"OriSt2i = {round(np.mean(ori_loss), 2)}")
-                logger.info(f"Levenshtein = {round(np.mean(L_distance), 2)}")
-                logger.info(f"robust = {robust_left} , {robust_right}")
-                logger.info(f"epsilon = {epsilon}")
-
-                end_time = time.time()
-                elapsed_time = end_time - start_time
-                hours, remainder = divmod(elapsed_time, 3600)
-                minutes, seconds = divmod(remainder, 60)
-                print(f"time cost: {int(hours)} hours, {int(minutes)} minutes, {int(seconds)} seconds")
-                logger.info(f"time cost: {int(hours)} hours, {int(minutes)} minutes, {int(seconds)} seconds")
-                logger.info(f"&" * 150)
+                n += 1
+            print("*" * 120)
+            logger.info(f"*" * 120)
+            logger.info(f"robust = {robust_re}")
+            logger.info(f"epsilon = {epsilon_re}")
+            logger.info(f"stop_early = {efficient_n}")
+            logger.info(f"futility = {efficient_m}")
+            logger.info(f"Non_AE = {Non_AE}")
+            logger.info(f"n = {n}")
+            logger.info(f"AdvSt2i = {round(np.mean(AdvSt2i), 2)}")
+            logger.info(f"OriSt2i = {round(np.mean(ori_loss), 2)}")
+            logger.info(f"Levenshtein = {round(np.mean(L_distance), 2)}")
+            logger.info(f"robust = {robust_left} , {robust_right}")
+            logger.info(f"epsilon = {epsilon}")
+
+            end_time = time.time()
+            elapsed_time = end_time - start_time
+            hours, remainder = divmod(elapsed_time, 3600)
+            minutes, seconds = divmod(remainder, 60)
+            print(f"time cost: {int(hours)} hours, {int(minutes)} minutes, {int(seconds)} seconds")
+            logger.info(f"time cost: {int(hours)} hours, {int(minutes)} minutes, {int(seconds)} seconds")
+            logger.info(f"&" * 150)