traffic-sign-classifier-robustness-testing / rt_search_based / strategies / pymoo_strategies.py
pymoo_strategies.py
Raw
from abc import abstractmethod
from typing import List

import numpy as np
import torch
from pymoo.algorithms.soo.nonconvex.ga import GA
from pymoo.core.algorithm import Algorithm
from pymoo.core.problem import ElementwiseProblem
from pymoo.factory import get_crossover, get_mutation, get_sampling
from pymoo.optimize import minimize
from rt_search_based.datasets.datasets import DatasetItem
from rt_search_based.fitness_functions.fitness_functions import FitnessFunction
from rt_search_based.models.classifiers import Classifier
from rt_search_based.strategies.strategies import Strategy
from rt_search_based.transformations import stickers
from rt_search_based.transformations.stickers import Color


class PyMooSingleObjectiveProblem(ElementwiseProblem):
    """
    pymoo problem definition. This is the core of the pymoo optimization.
    The problem is defined by the following attributes:
    - n_var: number of variables in the search space
    - n_obj: number of objectives
    - n_constr: number of constraints
    - xl: lower bound of the variables
    - xu: upper bound of the variables
    - type_var: type of the variables
    """

    def __init__(
        self,
        classifier: Classifier,
        fitness_function: FitnessFunction,
        dataset_item: DatasetItem,
        sticker_colors: List[Color],
        sticker_count: int = 1,
        use_constraints: bool = False,
    ):
        self.classifier = classifier
        self.fitness_function = fitness_function
        self.dataset_item = dataset_item
        self.sticker_colors = sticker_colors
        self.use_constraints = use_constraints

        _, self.img_height, self.img_width = self.dataset_item.image.size()

        super().__init__(
            n_var=5 * sticker_count,
            n_obj=1,
            n_constr=(2 * sticker_count + 1) * int(self.use_constraints),
            xl=np.zeros(sticker_count * 5),
            xu=np.array(
                [
                    self.img_width,
                    self.img_width,
                    self.img_height,
                    self.img_height,
                    len(self.sticker_colors) - 1,
                ]
                * sticker_count
            ),
            type_var=int,
        )

    def _evaluate(self, x, out, *args, **kwargs):
        properties = stickers.create_multi_sticker_properties(x, self.sticker_colors)
        img = stickers.add_multi_sticker_to_image(properties, self.dataset_item.image)

        # set fitness value as minimization target
        out["F"] = self.fitness_function(properties, self.dataset_item)
        # if specified also make use of constraints
        if self.use_constraints:
            # add the constraints for valid rectangles to the pymoo constraints list
            out["G"] = []
            for i in range(1, len(properties), 7):
                x1, y1, x2, y2, _, _, _ = properties[i : i + 7]
                out["G"].extend([x1 - x2, y1 - y2])
            # if specified also make the fooling of the model a constraint
            out["G"].append(self.passes_test(img))

    def passes_test(self, img):
        """
        Function to check if the model is fooled by the given stickered image.
        Used as a constraint in the pymoo optimization to only consider images
        that fool the model.
        """

        # Add extra dimension used for classifier (equivalent to batch_size 1)
        img = torch.unsqueeze(img, dim=0)
        predicted_class = self.classifier.get_predicted_class(img)

        correct_prediction = predicted_class == self.dataset_item.label
        return int(correct_prediction)


class PyMooStrategy(Strategy):
    def __init__(
        self,
        fitness_function: FitnessFunction,
        classifier: Classifier,
        sticker_colors: List[Color] = None,
        sticker_count: int = 1,
        need_constraints=False,
        **kwargs,
    ):

        if self.__class__.__name__ == "PyMooStrategy":
            raise TypeError

        super().__init__(
            fitness_function, classifier, sticker_colors, sticker_count, **kwargs
        )

        self.algorithm = self.get_new_algorithm()
        self.need_constraints = need_constraints

    @abstractmethod
    def get_new_algorithm(self) -> Algorithm:
        """return the algorithm that should be used"""

    def search_for_sticker(self, dataset_item: DatasetItem) -> torch.Tensor:
        """
        Function that searches for an optimal sticker in the given image.
        It creates a pymoo problem and optimizes it using the chosen pymoo algorithm,
        and then it returns the optimal sticker properties (solution).
        """

        problem = PyMooSingleObjectiveProblem(
            self.classifier,
            self.fitness_function,
            dataset_item,
            self.sticker_colors,
            self.sticker_count,
            self.need_constraints,
        )

        result = minimize(
            problem, self.algorithm, seed=1, save_history=True, verbose=True
        )
        solution = stickers.create_multi_sticker_properties(
            result.X, self.sticker_colors
        )

        # the code below checks if the solution fools the classifier
        img = stickers.add_multi_sticker_to_image(solution, dataset_item.image)

        # Add extra dimension used for classifier (equivalent to batch_size 1)
        img = torch.unsqueeze(img, dim=0)
        if self.classifier.get_predicted_class(img) == dataset_item.label:
            return torch.zeros_like(solution) - 1

        return solution


class PyMooGeneticStrategy(PyMooStrategy):
    def get_new_algorithm(self) -> Algorithm:
        return GA(
            sampling=get_sampling("int_random"),
            crossover=get_crossover("int_sbx"),
            mutation=get_mutation("int_pm"),
            # pop_size=100,
            # n_offsprings=2,
        )