diff --git a/episodes/fig/snakeviz-home.png b/episodes/fig/snakeviz-home.png new file mode 100644 index 0000000..65c3beb Binary files /dev/null and b/episodes/fig/snakeviz-home.png differ diff --git a/episodes/fig/snakeviz-worked-example-icicle.png b/episodes/fig/snakeviz-worked-example-icicle.png new file mode 100644 index 0000000..c621b54 Binary files /dev/null and b/episodes/fig/snakeviz-worked-example-icicle.png differ diff --git a/episodes/fig/snakeviz-worked-example-sunburst.png b/episodes/fig/snakeviz-worked-example-sunburst.png new file mode 100644 index 0000000..12abe3b Binary files /dev/null and b/episodes/fig/snakeviz-worked-example-sunburst.png differ diff --git a/episodes/files/pred-prey/predprey.py b/episodes/files/pred-prey/predprey.py new file mode 100644 index 0000000..5dee3a9 --- /dev/null +++ b/episodes/files/pred-prey/predprey.py @@ -0,0 +1,421 @@ +import math +import numpy as np +import matplotlib.pyplot as plt + +# Reproduction +REPRODUCE_PREY_PROB = 0.05 +REPRODUCE_PRED_PROB = 0.03 + +# Cohesion/Avoidance +SAME_SPECIES_AVOIDANCE_RADIUS = 0.035 +PREY_GROUP_COHESION_RADIUS = 0.2 + +# Predator/Prey/Grass interaction +PRED_PREY_INTERACTION_RADIUS = 0.3 +PRED_SPEED_ADVANTAGE = 3.0 +PRED_KILL_DISTANCE = 0.03 +GRASS_EAT_DISTANCE = 0.05 +GAIN_FROM_FOOD_PREY = 80 +GAIN_FROM_FOOD_PREDATOR = 100 +GRASS_REGROW_CYCLES = 20 +PRED_HUNGER_THRESH = 100 +PREY_HUNGER_THRESH = 100 + +# Simulation properties +DELTA_TIME = 0.001 +BOUNDS_WIDTH = 2.0 +MIN_POSITION = -1.0 +MAX_POSITION = 1.0 + +NEXT_PRED_ID = 1 +NEXT_PREY_ID = 1 + +class Prey: + def __init__(self): + global NEXT_PREY_ID + self.id = NEXT_PREY_ID + NEXT_PREY_ID += 1 + self.x = 0.0 + self.y = 0.0 + self.vx = 0.0 + self.vy = 0.0 + self.steer_x = 0.0 + self.steer_y = 0.0 + self.life = 0 + + def avoid_predators(self, predator_list): + # Reset this prey's steer force + self.steer_x = 0.0 + self.steer_y = 0.0 + + # Add a steering factor away from each predator. Strength increases with closeness. + for predator in predator_list: + # Fetch location of predator + predator_x = self.x; + predator_y = self.y; + + # Check if the two predators are within interaction radius + dx = self.x - predator.x + dy = self.y - predator.y + distance = math.sqrt(dx * dx + dy * dy) + + if distance < PRED_PREY_INTERACTION_RADIUS: + # Steer the prey away from the predator + self.steer_x += (PRED_PREY_INTERACTION_RADIUS / distance) * dx + self.steer_y += (PRED_PREY_INTERACTION_RADIUS / distance) * dy + + def flock(self, prey_list): + group_centre_x = 0.0 + group_centre_y = 0.0 + group_velocity_x = 0.0 + group_velocity_y = 0.0 + avoid_velocity_x = 0.0 + avoid_velocity_y = 0.0 + group_centre_count = 0 + + for other in prey_list: + dx = self.x - other.x + dy = self.y - other.y + separation = math.sqrt(dx * dx + dy * dy) + + if separation < PREY_GROUP_COHESION_RADIUS and self.id != other.id: + group_centre_x += other.x + group_centre_y += other.y + group_centre_count += 1 + + # Avoidance behaviour + if separation < SAME_SPECIES_AVOIDANCE_RADIUS: + # Was a check for separation > 0 in original - redundant? + avoid_velocity_x += SAME_SPECIES_AVOIDANCE_RADIUS / separation * dx + avoid_velocity_y += SAME_SPECIES_AVOIDANCE_RADIUS / separation * dy + + # Compute group centre as the average of the nearby prey positions and a velocity to move towards the group centre + if group_centre_count > 0: + group_centre_x /= group_centre_count + group_centre_y /= group_centre_count + group_velocity_x = group_centre_x - self.x + group_velocity_y = group_centre_y - self.y + + self.steer_x += group_velocity_x + avoid_velocity_x + self.steer_y += group_velocity_y + avoid_velocity_y + + def move(self): + # Integrate steering forces and cap velocity + self.vx += self.steer_x + self.vy += self.steer_y + + speed = math.sqrt(self.vx * self.vx + self.vy * self.vy) + if speed > 1.0: + self.vx /= speed + self.vy /= speed + + # Integrate velocity + self.x += self.vx * DELTA_TIME + self.y += self.vy * DELTA_TIME + + # Bound the position within the environment - can this be moved + self.x = max(self.x, MIN_POSITION) + self.x = min(self.x, MAX_POSITION) + self.y = max(self.y, MIN_POSITION) + self.y = min(self.y, MAX_POSITION) + + # Reduce life by one unit of energy + self.life -= 1 + + def eaten_or_starve(self, predator_list): + predator_index = -1 + closest_pred = PRED_KILL_DISTANCE + + # Iterate predator_location messages to find the closest predator + for i in range(len(predator_list)): + predator = predator_list[i] + if predator.life < PRED_HUNGER_THRESH: + # Check if the two predators are within interaction radius + dx = self.x - predator.x + dy = self.y - predator.y + distance = math.sqrt(dx * dx + dy * dy) + + if distance < closest_pred: + predator_index = i + closest_pred = distance + + if predator_index >= 0: + predator_list[predator_index].life += GAIN_FROM_FOOD_PREDATOR + return True + + # If the life has reduced to 0 then the prey should die or starvation + if self.life < 1: + return True + return False + + def reproduce(self): + if np.random.uniform() < REPRODUCE_PREY_PROB: + self.life /= 2 + + child = Prey() + child.x = np.random.uniform() * BOUNDS_WIDTH - BOUNDS_WIDTH / 2.0 + child.y = np.random.uniform() * BOUNDS_WIDTH - BOUNDS_WIDTH / 2.0 + child.vx = np.random.uniform() * 2 - 1 + child.vy = np.random.uniform() * 2 - 1 + child.life = self.life + + +class Predator: + def __init__(self): + global NEXT_PRED_ID + self.id = NEXT_PRED_ID + NEXT_PRED_ID += 1 + self.x = 0.0 + self.y = 0.0 + self.vx = 0.0 + self.vy = 0.0 + self.steer_x = 0.0 + self.steer_y = 0.0 + self.life = 0 + + def follow_prey(self, prey_list): + # Find the closest prey by iterating the prey_location messages + closest_prey_x = 0.0 + closest_prey_y = 0.0 + closest_prey_distance = PRED_PREY_INTERACTION_RADIUS + is_a_prey_in_range = 0 + + for prey in prey_list: + # Check if prey is within sight range of predator + dx = self.x - prey.x + dy = self.y - prey.y + separation = math.sqrt(dx * dx + dy * dy) + + if separation < closest_prey_distance: + closest_prey_x = prey.x + closest_prey_y = prey.y + closest_prey_distance = separation + is_a_prey_in_range = 1 + + # If there was a prey in range, steer the predator towards it + if is_a_prey_in_range: + self.steer_x = closest_prey_x - self.x + self.steer_y = closest_prey_y - self.y + + + def avoid_predators(self, predator_list): + # Fetch this predator's position + avoid_velocity_x = 0.0 + avoid_velocity_y = 0.0 + + # Add a steering factor away from each other predator. Strength increases with closeness. + for other in predator_list: + # Check if the two predators are within interaction radius + dx = self.x - other.x + dy = self.y - other.y + separation = math.sqrt(dx * dx + dy * dy) + + if separation < SAME_SPECIES_AVOIDANCE_RADIUS and separation > 0.0 and self.id != other.id: + avoid_velocity_x += SAME_SPECIES_AVOIDANCE_RADIUS / separation * dx + avoid_velocity_y += SAME_SPECIES_AVOIDANCE_RADIUS / separation * dy + + self.steer_x += avoid_velocity_x + self.steer_y += avoid_velocity_y + + def move(self): + # Integrate steering forces and cap velocity + self.vx += self.steer_x + self.vy += self.steer_y + + speed = math.sqrt(self.vx * self.vx + self.vy * self.vy) + if speed > 1.0: + self.vx /= speed + self.vy /= speed + + # Integrate velocity + self.x += self.vx * DELTA_TIME * PRED_SPEED_ADVANTAGE + self.y += self.vy * DELTA_TIME * PRED_SPEED_ADVANTAGE + + # Bound the position within the environment + self.x = max(self.x, MIN_POSITION) + self.x = min(self.x, MAX_POSITION) + self.y = max(self.y, MIN_POSITION) + self.y = min(self.y, MAX_POSITION) + + # Reduce life by one unit of energy + self.life -= 1 + + def starve(self): + # Did the predator starve? + if self.life < 1: + return True + return False + + def reproduce(self): + if np.random.uniform() < REPRODUCE_PRED_PROB: + self.life /= 2 + + child = Predator() + child.x = np.random.uniform() * BOUNDS_WIDTH - BOUNDS_WIDTH / 2.0 + child.y = np.random.uniform() * BOUNDS_WIDTH - BOUNDS_WIDTH / 2.0 + child.vx = np.random.uniform() * 2 - 1 + child.vy = np.random.uniform() * 2 - 1 + child.life = self.life + return child + +class Grass: + def __init__(self): + self.x = 0.0 + self.y = 0.0 + self.dead_cycles = 0 + self.available = 1 + + def grow(self): + new_dead_cycles = self.dead_cycles + 1 + if self.dead_cycles == GRASS_REGROW_CYCLES: + self.dead_cycles = 0 + self.available = 1 + + if self.available == 0: + self.dead_cycles = new_dead_cycles + + + def eaten(self, prey_list): + if self.available: + prey_index = -1 + closest_prey = GRASS_EAT_DISTANCE + + # Iterate prey_location messages to find the closest prey + for i in range(len(prey_list)): + prey = prey_list[i] + if prey.life < PREY_HUNGER_THRESH: + # Check if they are within interaction radius + dx = self.x - prey.x + dy = self.y - prey.y + distance = math.sqrt(dx*dx + dy*dy) + + if distance < closest_prey: + prey_index = i + closest_prey = distance + + if prey_index >= 0: + # Add grass eaten message + prey_list[prey_index].life += GAIN_FROM_FOOD_PREY + + # Update grass agent variables + self.dead_cycles = 0 + self.available = 0 + +class Model: + + def __init__(self, steps = 250): + self.steps = steps + self.num_prey = 200 + self.num_predators = 50 + self.num_grass = 5000 + + def _init_population(self): + # Initialise prey agents + self.prey = [] + for i in range(self.num_prey): + p = Prey() + p.x = np.random.uniform(-1.0, 1.0) + p.y = np.random.uniform(-1.0, 1.0) + p.vx = np.random.uniform(-1.0, 1.0) + p.vy = np.random.uniform(-1.0, 1.0) + p.life = np.random.randint(10, 50) + self.prey.append(p) + + # Initialise predator agents + self.predators = [] + for i in range(self.num_predators): + p = Predator() + p.x = np.random.uniform(-1.0, 1.0) + p.y = np.random.uniform(-1.0, 1.0) + p.vx = np.random.uniform(-1.0, 1.0) + p.vy = np.random.uniform(-1.0, 1.0) + p.life = np.random.randint(10, 15) + self.predators.append(p) + + # Initialise grass agents + self.grass = [] + for i in range(self.num_grass): + g = Grass() + g.x = np.random.uniform(-1.0, 1.0) + g.y = np.random.uniform(-1.0, 1.0) + self.grass.append(g) + + def _step(self): + ## Shuffle agent list order to avoid bias + np.random.shuffle(self.predators) # todo, this probably doesn't like Python lists + np.random.shuffle(self.prey) + + for p in self.predators: + p.follow_prey(self.prey) + for p in self.prey: + p.avoid_predators(self.predators) + + for p in self.prey: + p.flock(self.prey) + for p in self.predators: + p.avoid_predators(self.predators) + + for p in self.prey: + p.move() + for p in self.predators: + p.move() + + + for g in self.grass: + g.eaten(self.prey) + + self.prey = [p for p in self.prey if not p.eaten_or_starve(self.predators)] + self.predators = [p for p in self.predators if not p.starve()] + + children = [] + for p in self.prey: + c = p.reproduce() + if c: + children.append(c) + self.predators.extend(children) + children = [] + for p in self.predators: + c = p.reproduce() + if c: + children.append(c) + self.predators.extend(children) + for g in self.grass: + g.grow() + + def _init_log(self): + self.prey_log = [len(self.prey)] + self.predator_log = [len(self.predators)] + self.grass_log = [sum(g.available for g in self.grass)/20] + + def _log(self): + self.prey_log.append(len(self.prey)) + self.predator_log.append(len(self.predators)) + self.grass_log.append(sum(g.available for g in self.grass)/20) + + def _plot(self): + plt.figure(figsize=(16,10)) + plt.rcParams.update({'font.size': 18}) + plt.xlabel("Step") + plt.ylabel("Population") + plt.plot(range(0, len(self.prey_log)), self.prey_log, 'b', label="Prey") + plt.plot(range(0, len(self.predator_log)), self.predator_log, 'r', label="Predators") + plt.plot(range(0, len(self.grass_log)), self.grass_log, 'g', label="Grass/20") + plt.legend() + plt.savefig('predprey_out.png') + + def run(self): + # init + self._init_population() + self._init_log() + # execute + for i in range(self.steps): + self._step() + self._log() + # plot graph of results + self._plot() + + + + +model = Model() +model.run() \ No newline at end of file diff --git a/episodes/files/snakeviz-worked-example/example.py b/episodes/files/snakeviz-worked-example/example.py new file mode 100644 index 0000000..13d9d73 --- /dev/null +++ b/episodes/files/snakeviz-worked-example/example.py @@ -0,0 +1,36 @@ +import time + +""" +This is a synthetic program intended to produce a clear profile with cProfile/snakeviz +Method names, constructed from a hex digit and a number clearly denote their position in the hierarchy. +""" +def a_1(): + for i in range(3): + b_1() + time.sleep(1) + b_2() + + +def b_1(): + c_1() + c_2() + +def b_2(): + time.sleep(1) + +def c_1(): + time.sleep(0.5) + + +def c_2(): + time.sleep(0.3) + d_1() + +def d_1(): + time.sleep(0.1) + + + + +# Entry Point +a_1() \ No newline at end of file diff --git a/episodes/files/snakeviz-worked-example/out.prof b/episodes/files/snakeviz-worked-example/out.prof new file mode 100644 index 0000000..78fb9f3 Binary files /dev/null and b/episodes/files/snakeviz-worked-example/out.prof differ diff --git a/episodes/files/travelling-sales/travellingsales.py b/episodes/files/travelling-sales/travellingsales.py new file mode 100644 index 0000000..2ac9557 --- /dev/null +++ b/episodes/files/travelling-sales/travellingsales.py @@ -0,0 +1,48 @@ +""" +Naive brute force travelling salesperson +python travellingsales.py +""" + +import itertools +import math +import random +import sys + +def distance(point1, point2): + return math.sqrt((point2[0] - point1[0])**2 + (point2[1] - point1[1])**2) + +def total_distance(points, order): + total = 0 + for i in range(len(order) - 1): + total += distance(points[order[i]], points[order[i + 1]]) + return total + distance(points[order[-1]], points[order[0]]) + +def traveling_salesman_brute_force(points): + min_distance = float('inf') + min_path = None + for order in itertools.permutations(range(len(points))): + d = total_distance(points, order) + if d < min_distance: + min_distance = d + min_path = order + return min_path, min_distance + +# Argument parsing +if len(sys.argv) != 2: + print("Script expects 1 positive integer argument, %u found."%(len(sys.argv) - 1)) + sys.exit() +cities_len = int(sys.argv[1]) +if cities_len < 1: + print("Script expects 1 positive integer argument, %s converts < 1."%(sys.argv[1])) + sys.exit() +# Define the cities as (x, y) coordinates +random.seed(12) # Fixed random for consistency +cities = [(0,0)] +for i in range(cities_len): + cities.append((random.uniform(-1, 1), random.uniform(-1, 1))) + +# Find the shortest path +shortest_path, min_distance = traveling_salesman_brute_force(cities) +print("Cities:", cities_len) +print("Shortest Path:", shortest_path) +print("Shortest Distance:", min_distance) diff --git a/episodes/profiling-functions.md b/episodes/profiling-functions.md index d1d629b..b7766c8 100644 --- a/episodes/profiling-functions.md +++ b/episodes/profiling-functions.md @@ -6,12 +6,301 @@ exercises: 0 :::::::::::::::::::::::::::::::::::::: questions -- TODO +- When is function level profiling appropriate? +- How can `cProfile` and `snakeviz` be used to profile a Python program? +- How are the outputs from function level profiling interpreted? :::::::::::::::::::::::::::::::::::::::::::::::: ::::::::::::::::::::::::::::::::::::: objectives -- TODO +- execute a Python program via `cProfile` to collect profiling information about a Python program’s execution +- use `snakeviz` to visualise profiling information output by `cProfile` +- interpret `snakeviz` views, to identify the functions where time is being spent during a program’s execution -:::::::::::::::::::::::::::::::::::::::::::::::: \ No newline at end of file +:::::::::::::::::::::::::::::::::::::::::::::::: + +## Introduction + +Software is typically comprised of a hierarchy of function calls, both functions written by the developer and those used from the core language and third party packages. + + +Function-level profiling analyses where time is being spent with respect to functions. Typically function-level profiling will calculate the number of times each function is called and the total time spent executing each function, inclusive and exclusive of child function calls. + + +This allows functions that occupy a disproportionate amount of the total runtime to be quickly identified and investigated. + + +In this episode we will cover the usage of the function-level profiler `cProfile`, how it's output can be visualised with `snakeviz` and how the output can be interpreted. + +## cProfile + + +[`cProfile`](https://docs.python.org/3/library/profile.html#instant-user-s-manual) is a function-level profiler provided as part of the Python standard library. + + +It can be called directly within your Python code as an imported package, however it's easier to use it's script interface: + +```sh +python -m cProfile -o