--- title: Examples keywords: fastai sidebar: home_sidebar summary: "Examples of the PCT library in use." description: "Examples of the PCT library in use." nb_path: "nbs/00_examples.ipynb" ---
%load_ext autoreload
%autoreload 2
#import sys
#sys.path.append("..")
import gym
render=False
runs=1
render=True
runs=2000
Cartpole is an Open AI gym environment for the inverted pendulum problem. The goal is to keep the pole balanced, by moving the cart left or right.
The environment provides observations (perceptions) for the state of the cart and pole.
0 - Cart Position
1 - Cart Velocity
2 - Pole Angle
3 - Pole Angular Velocity
It takes one value, of 0 or 1, for applying a force to the left or right, respectively.
The PCT solution is a four-level hierarchy for controlling the perceptions at goal values. Only one goal reference is manually set, the highest level which is the pole angle of 0.
This example shows how a perceptual control hierarchy can be implemented with this library.
import matplotlib.pyplot as plt
import numpy as np
from pct.hierarchy import PCTHierarchy
from pct.putils import FunctionsList
from pct.environments import CartPoleV1
from pct.functions import IndexedParameter
from pct.functions import Integration
from pct.functions import GreaterThan
from pct.functions import PassOn
Create a hierarchy of 4 levels each with one node.
cartpole_hierarchy = PCTHierarchy(levels=4, cols=1, name="cartpoleh", build=False)
namespace=cartpole_hierarchy.namespace
cartpole_hierarchy.get_node(0, 0).name = 'cart_velocity_node'
cartpole_hierarchy.get_node(1, 0).name = 'cart_position_node'
cartpole_hierarchy.get_node(2, 0).name = 'pole_velocity_node'
cartpole_hierarchy.get_node(3, 0).name = 'pole_angle_node'
#FunctionsList.getInstance().report()
#cartpole_hierarchy.summary(build=True)
Create the Cartpole gym environment function. This will apply the "action" output from the hierarchy and provide the new observations.
cartpole = CartPoleV1(name="CartPole-v1", render=render, namespace=namespace)
Create functions for each of the observation parameters of the Cartpole environment. Insert them into the hierarchy at the desired places.
cartpole_hierarchy.insert_function(level=0, col=0, collection="perception", function=IndexedParameter(index=1, name="cart_velocity", links=[cartpole], namespace=namespace))
cartpole_hierarchy.insert_function(level=1, col=0, collection="perception", function=IndexedParameter(index=0, name="cart_position", links=[cartpole], namespace=namespace))
cartpole_hierarchy.insert_function(level=2, col=0, collection="perception", function=IndexedParameter(index=3, name="pole_velocity", links=[cartpole], namespace=namespace))
cartpole_hierarchy.insert_function(level=3, col=0, collection="perception", function=IndexedParameter(index=2, name="pole_angle", links=[cartpole], namespace=namespace))
Link the references to the outputs of the level up.
cartpole_hierarchy.insert_function(level=0, col=0, collection="reference", function=PassOn(name="cart_velocity_reference", links=['proportional1'], namespace=namespace))
cartpole_hierarchy.insert_function(level=1, col=0, collection="reference", function=PassOn(name="cart_position_reference", links=['proportional2'], namespace=namespace))
cartpole_hierarchy.insert_function(level=2, col=0, collection="reference", function=PassOn(name="pole_velocity_reference", links=['proportional3'], namespace=namespace))
Set the highest level reference.
top = cartpole_hierarchy.get_function(level=3, col=0, collection="reference")
top.set_name("pole_angle_reference")
top.set_value(0)
Link the output of the hierarchy back to the Cartpole environment.
cartpole_hierarchy.summary(build=True)
cartpole_hierarchy.insert_function(level=0, col=0, collection="output", function=Integration(gain=-0.05, slow=4, name="force", links='subtract', namespace=namespace))
Set the names and gains of the output functions. This also shows another way of getting a function, by name.
FunctionsList.getInstance().get_function(namespace=namespace, name="proportional3").set_name("pole_angle_output")
FunctionsList.getInstance().get_function(namespace=namespace, name="pole_angle_output").set_property('gain', 3.5)
FunctionsList.getInstance().get_function(namespace=namespace, name="proportional2").set_name("pole_velocity_output")
FunctionsList.getInstance().get_function(namespace=namespace, name="pole_velocity_output").set_property('gain', 0.5)
FunctionsList.getInstance().get_function(namespace=namespace, name="proportional1").set_name("cart_position_output")
FunctionsList.getInstance().get_function(namespace=namespace, name="cart_position_output").set_property('gain', 2)
Add a post function to convert the output to 1 or 0 as required by the Cartpole environment.
greaterthan = GreaterThan(threshold=0, upper=1, lower=0, links='force', namespace=namespace)
cartpole_hierarchy.add_postprocessor(greaterthan)
Add the cartpole function as one that is executed before the actual hierarchy.
cartpole_hierarchy.add_preprocessor(cartpole)
Set the output of the hierachy as the action input to the Cartpole environment.
cartpole.add_link(greaterthan)
Sit back and observe the brilliance of your efforts.
cartpole_hierarchy.set_order("Down")
cartpole_hierarchy.summary()
cartpole_hierarchy.draw(font_size=10, figsize=(8,12), move={'CartPole-v1': [-0.075, 0]}, node_size=1000, node_color='red')
cartpole_hierarchy.save("cartpole.json")
import networkx as nx
gr = cartpole_hierarchy.graph()
print(nx.info(gr))
print(gr.nodes())
Run the hierarchy for 500 steps.
cartpole_hierarchy.run(1,verbose=False)
cartpole_hierarchy.run(runs,verbose=False)
cartpole.close()