Enhancements

Author: ahmedfgad

pygad/cnn/cnn.py

@@ -1,6 +1,7 @@
 import numpy
 import functools
 import logging
+from ..helper.activations import sigmoid, relu, softmax
 
 """
 Convolutional neural network implementation using NumPy
@@ -14,56 +15,8 @@
 # Supported activation functions by the cnn.py module.
 supported_activation_functions = ("sigmoid", "relu", "softmax")
 
-def sigmoid(sop):
-
-    """
-    Applies the sigmoid function.
-
-    sop: The input to which the sigmoid function is applied.
-
-    Returns the result of the sigmoid function.
-    """
-
-    if type(sop) in [list, tuple]:
-        sop = numpy.array(sop)
-
-    return 1.0 / (1 + numpy.exp(-1 * sop))
-
-def relu(sop):
-
-    """
-    Applies the rectified linear unit (ReLU) function.
-
-    sop: The input to which the relu function is applied.
-
-    Returns the result of the ReLU function.
-    """
-
-    if not (type(sop) in [list, tuple, numpy.ndarray]):
-        if sop < 0:
-            return 0
-        else:
-            return sop
-    elif type(sop) in [list, tuple]:
-        sop = numpy.array(sop)
-
-    result = sop
-    result[sop < 0] = 0
-
-    return result
-
-def softmax(layer_outputs):
-
-    """
-    Applies the sotmax function.
-
-    sop: The input to which the softmax function is applied.
-
-    Returns the result of the softmax function.
-    """
-    return layer_outputs / (numpy.sum(layer_outputs) + 0.000001)
-
-def layers_weights(model, initial=True):
+def layers_weights(model, 
+ initial=True):
 
     """
     Creates a list holding the weights of all layers in the CNN.

pygad/helper/activations.py

@@ -0,0 +1,47 @@
+import numpy
+
+def sigmoid(sop):
+    """
+    Applies the sigmoid function.
+
+    sop: The input to which the sigmoid function is applied.
+
+    Returns the result of the sigmoid function.
+    """
+
+    if type(sop) in [list, tuple]:
+        sop = numpy.array(sop)
+
+    return 1.0 / (1 + numpy.exp(-1 * sop))
+
+def relu(sop):
+    """
+    Applies the ReLU function.
+
+    sop: The input to which the relu function is applied.
+
+    Returns the result of the ReLU function.
+    """
+
+    if not (type(sop) in [list, tuple, numpy.ndarray]):
+        if sop < 0:
+            return 0
+        else:
+            return sop
+    elif type(sop) in [list, tuple]:
+        sop = numpy.array(sop)
+
+    result = sop
+    result[sop < 0] = 0
+
+    return result
+
+def softmax(layer_outputs):
+    """
+    Applies the softmax function.
+
+    sop: The input to which the softmax function is applied.
+
+    Returns the result of the softmax function.
+    """
+    return layer_outputs / (numpy.sum(layer_outputs) + 0.000001)

pygad/nn/nn.py

@@ -1,5 +1,6 @@
 import numpy
 import functools
+from ..helper.activations import sigmoid, relu, softmax
 
 """
 This project creates a neural network where the architecture has input and dense layers only. More layers will be added in the future. 
@@ -140,57 +141,7 @@ def layers_activations(last_layer):
     activations.reverse()
     return activations
 
-def sigmoid(sop):
-
-    """
-    Applies the sigmoid function.
-
-    sop: The input to which the sigmoid function is applied.
-
-    Returns the result of the sigmoid function.
-    """
-
-    if type(sop) in [list, tuple]:
-        sop = numpy.array(sop)
-
-    return 1.0 / (1 + numpy.exp(-1 * sop))
-
-def relu(sop):
-
-    """
-    Applies the rectified linear unit (ReLU) function.
-
-    sop: The input to which the relu function is applied.
-
-    Returns the result of the ReLU function.
-    """
-
-    if not (type(sop) in [list, tuple, numpy.ndarray]):
-        if sop < 0:
-            return 0
-        else:
-            return sop
-    elif type(sop) in [list, tuple]:
-        sop = numpy.array(sop)
-
-    result = sop
-    result[sop < 0] = 0
-
-    return result
-
-def softmax(layer_outputs):
-
-    """
-    Applies the sotmax function.
-
-    sop: The input to which the softmax function is applied.
-
-    Returns the result of the softmax function.
-    """
-    return layer_outputs / (numpy.sum(layer_outputs) + 0.000001)
-
-def train(num_epochs, 
-          last_layer, 
+def train(num_epochs,           last_layer, 
           data_inputs, 
           data_outputs,
           problem_type="classification",

pygad/utils/crossover.py

@@ -198,13 +198,11 @@ def uniform_crossover(self, parents, offspring_size):
                 # Index of the second parent to mate.
                 parent2_idx = (k+1) % parents.shape[0]
 
-            for gene_idx in range(offspring_size[1]):
-                if (genes_sources[k, gene_idx] == 0):
-                    # The gene will be copied from the first parent if the current gene index is 0.
-                    offspring[k, gene_idx] = parents[parent1_idx, gene_idx]
-                elif (genes_sources[k, gene_idx] == 1):
-                    # The gene will be copied from the second parent if the current gene index is 1.
-                    offspring[k, gene_idx] = parents[parent2_idx, gene_idx]
+            # The gene will be copied from the first parent if the current gene index is 0.
+            # The gene will be copied from the second parent if the current gene index is 1.
+            offspring[k, :] = numpy.where(genes_sources[k] == 0, 
+                                          parents[parent1_idx, :], 
+                                          parents[parent2_idx, :])
 
             if self.allow_duplicate_genes == False:
                 if self.gene_space is None:
@@ -268,6 +266,8 @@ def scattered_crossover(self, parents, offspring_size):
                 # Index of the second parent to mate.
                 parent2_idx = (k+1) % parents.shape[0]
 
+            # The gene will be copied from the first parent if the current gene index is 0.
+            # The gene will be copied from the second parent if the current gene index is 1.
             offspring[k, :] = numpy.where(genes_sources[k] == 0, 
                                           parents[parent1_idx, :], 
                                           parents[parent2_idx, :])

pygad/utils/engine.py

@@ -6,12 +6,12 @@
 class GAEngine:
 
     def round_genes(self, solutions):
-        for gene_idx in range(self.num_genes):
-            if self.gene_type_single:
-                if not self.gene_type[1] is None:
-                    solutions[:, gene_idx] = numpy.round(solutions[:, gene_idx],
-                                                         self.gene_type[1])
-            else:
+        if self.gene_type_single:
+            if not self.gene_type[1] is None:
+                solutions = numpy.round(numpy.asarray(solutions, dtype=self.gene_type[0]),
+                                        self.gene_type[1])
+        else:
+            for gene_idx in range(self.num_genes):
                 if not self.gene_type[gene_idx][1] is None:
                     solutions[:, gene_idx] = numpy.round(numpy.asarray(solutions[:, gene_idx],
                                                                        dtype=self.gene_type[gene_idx][0]),
@@ -77,6 +77,7 @@ def initialize_population(self,
                                                                                              sample_size=1)
 
         # 2) Change the data type and round all genes within the initial population.
+        # This step is necessary before applying the gene constraints since the right gene value must be used for accuracy.
         self.population = self.change_population_dtype_and_round(self.population)
 
         # Note that gene_constraint is not validated yet.

pygad/utils/parent_selection.py

@@ -112,12 +112,14 @@ def tournament_selection(self, fitness, num_parents):
         parents = self.initialize_parents_array((num_parents, self.population.shape[1]))
         parents_indices = []
 
+        rank_lookup = {sol_idx: rank for rank, sol_idx in enumerate(fitness_sorted)}
+
         for parent_num in range(num_parents):
             # Generate random indices for the candidate solutions.
             rand_indices = numpy.random.randint(low=0, high=len(fitness), size=self.K_tournament)
 
             # Find the rank of the candidate solutions. The lower the rank, the better the solution.
-            rand_indices_rank = [fitness_sorted.index(rand_idx) for rand_idx in rand_indices]
+            rand_indices_rank = [rank_lookup[rand_idx] for rand_idx in rand_indices]
             # Select the solution with the lowest rank as a parent.
             selected_parent_idx = rand_indices_rank.index(min(rand_indices_rank))
 
@@ -196,17 +198,10 @@ def wheel_cumulative_probs(self, probs, num_parents):
         probs_start = numpy.zeros(probs.shape, dtype=float) # An array holding the start values of the ranges of probabilities.
         probs_end = numpy.zeros(probs.shape, dtype=float) # An array holding the end values of the ranges of probabilities.
 
-        curr = 0.0
-
-        # Calculating the probabilities of the solutions to form a roulette wheel.
-        for _ in range(probs.shape[0]):
-            min_probs_idx = numpy.where(probs == numpy.min(probs))[0][0]
-            probs_start[min_probs_idx] = curr
-            curr = curr + probs[min_probs_idx]
-            probs_end[min_probs_idx] = curr
-            # Replace 99999999999 by float('inf')
-            # probs[min_probs_idx] = 99999999999
-            probs[min_probs_idx] = float('inf')
+        sorted_indices = numpy.argsort(probs)
+        cumulative = numpy.cumsum(probs[sorted_indices])
+        probs_start[sorted_indices] = numpy.concatenate([[0.0], cumulative[:-1]])
+        probs_end[sorted_indices] = cumulative
 
         # Selecting the best individuals in the current generation as parents for producing the offspring of the next generation.
         parents = self.initialize_parents_array((num_parents, self.population.shape[1]))
@@ -248,18 +243,8 @@ def stochastic_universal_selection(self, fitness, num_parents):
 
         probs = fitness / fitness_sum
 
-        probs_start = numpy.zeros(probs.shape, dtype=float) # An array holding the start values of the ranges of probabilities.
-        probs_end = numpy.zeros(probs.shape, dtype=float) # An array holding the end values of the ranges of probabilities.
-
-        curr = 0.0
-
-        # Calculating the probabilities of the solutions to form a roulette wheel.
-        for _ in range(probs.shape[0]):
-            min_probs_idx = numpy.where(probs == numpy.min(probs))[0][0]
-            probs_start[min_probs_idx] = curr
-            curr = curr + probs[min_probs_idx]
-            probs_end[min_probs_idx] = curr
-            probs[min_probs_idx] = float('inf')
+        probs_start, probs_end, parents = self.wheel_cumulative_probs(probs=probs.copy(), 
+                                                                      num_parents=num_parents)
 
         pointers_distance = 1.0 / self.num_parents_mating # Distance between different pointers.
         first_pointer = numpy.random.uniform(low=0.0,