We are building a natural-language controller for a simulated 2D pick-and-place robot. The LLM translates commands like "move the red box to the left shelf" into discrete actions that update a simple physics state. You can run the whole stack in a single Python file, then swap the simulator for a real ROS2 node when you are ready.
What you'll need
- Python 3.10 or newer
- The OpenAI SDK:
pip install openai - An Oxlo.ai API key from https://portal.oxlo.ai. The free tier includes 60 requests per day, which is enough to test this controller.
Step 1: Connect to Oxlo.ai and verify the client
First, I initialize the OpenAI-compatible client pointing at Oxlo.ai and make a quick sanity call. I use qwen-3-32b because it is optimized for agentic workflows.
from openai import OpenAI
client = OpenAI(
base_url="https://api.oxlo.ai/v1",
api_key="YOUR_OXLO_API_KEY",
)
response = client.chat.completions.create(
model="qwen-3-32b",
messages=[
{"role": "system", "content": "You are a helpful robotics assistant."},
{"role": "user", "content": "Reply with OK if you are online."},
],
)
print(response.choices[0].message.content)Step 2: Model the robot world and action space
Next, I define the environment. The robot arm lives on a 10 by 10 grid. It can move to integer coordinates, grip when near an object, and release at a destination. The state is just a Python dictionary.
WORLD = {
"bounds": (10, 10),
"robot": {"position": [0, 0], "gripper": "empty"},
"objects": {
"red_box": {"position": [3, 3], "state": "on_table"},
"blue_cube": {"position": [7, 2], "state": "on_table"},
},
"bins": {
"left_bin": {"position": [1, 9]},
"top_bin": {"position": [5, 9]},
},
}
def describe_state(world):
lines = [
f"Robot is at {world['robot']['position']}, gripper is {world['robot']['gripper']}.",
"Objects:",
]
for name, obj in world["objects"].items():
lines.append(f"- {name}: {obj['position']} ({obj['state']})")
lines.append("Bins:")
for name, bin_ in world["bins"].items():
lines.append(f"- {name}: {bin_['position']}")
return "\n".join(lines)Step 3: Write the system prompt
The system prompt grounds the model in the robot's valid actions and forces it to output one command per line. Keeping the prompt explicit reduces hallucinated commands.
SYSTEM_PROMPT = """
You are the controller for a 2D pick-and-place robot.
The world is a 10 by 10 grid.
Valid actions, one per line:
- MOVE_TO x y
- GRIP
- RELEASE
Rules:
- You must MOVE_TO within 1 unit of an object before you GRIP it.
- After you GRIP, the object is attached to the gripper.
- MOVE_TO the destination bin before RELEASE.
- Do not explain. Output only the action list.
- If the command is impossible, output exactly: FAIL: reason
"""Step 4: Build the planner
This function formats the current state and the user command, sends them to Oxlo.ai, and parses the returned plan into a list of actions.
from openai import OpenAI
client = OpenAI(base_url="https://api.oxlo.ai/v1", api_key="YOUR_OXLO_API_KEY")
def plan_actions(world, user_command):
state_text = describe_state(world)
user_message = f"""Current state:
{state_text}
User command: {user_command}
Actions:"""
response = client.chat.completions.create(
model="qwen-3-32b",
messages=[
{"role": "system", "content": SYSTEM_PROMPT},
{"role": "user", "content": user_message},
],
)
raw = response.choices[0].message.content.strip()
actions = [line.strip() for line in raw.splitlines() if line.strip()]
return actionsStep 5: Simulate execution and safety checks
Before we trust the LLM, I add a simple validator. If the model outputs an invalid coordinate or tries to grip from too far away, we catch it and halt. This keeps the simulator from drifting into nonsense states.
import math
def execute_actions(world, actions):
robot = world["robot"]
for act in actions:
if act.startswith("FAIL:"):
print(f"Planner reported failure: {act}")
return
parts = act.split()
cmd = parts[0]
if cmd == "MOVE_TO":
x, y = int(parts[1]), int(parts[2])
if not (0 <= x <= world["bounds"][0] and 0 <= y <= world["bounds"][1]):
print(f"Invalid move: {x},{y}")
return
robot["position"] = [x, y]
print(f"Moved to {x},{y}")
elif cmd == "GRIP":
if robot["gripper"] != "empty":
print("Gripper already full")
return
held = None
for name, obj in world["objects"].items():
if obj["state"] != "on_table":
continue
dist = math.dist(robot["position"], obj["position"])
if dist <= 1.5:
held = name
break
if held is None:
print("No object in range to grip")
return
robot["gripper"] = held
world["objects"][held]["state"] = "in_gripper"
print(f"Gripped {held}")
elif cmd == "RELEASE":
if robot["gripper"] == "empty":
print("Nothing to release")
return
held = robot["gripper"]
robot["gripper"] = "empty"
world["objects"][held]["state"] = "in_bin"
world["objects"][held]["position"] = list(robot["position"])
print(f"Released {held} at {world['objects'][held]['position']}")
else:
print(f"Unknown action: {act}")
returnRun it
Because Oxlo.ai uses flat per-request pricing (see https://oxlo.ai/pricing), running this iterative planner costs the same whether the state description is ten lines or a hundred. Here is the complete script and the output for the command "pick up the blue cube and drop it in the top bin".
from openai import OpenAI
import math
client = OpenAI(base_url="https://api.oxlo.ai/v1", api_key="YOUR_OXLO_API_KEY")
WORLD = {
"bounds": (10, 10),
"robot": {"position": [0, 0], "gripper": "empty"},
"objects": {
"red_box": {"position": [3, 3], "state": "on_table"},
"blue_cube": {"position": [7, 2], "state": "on_table"},
},
"bins": {
"left_bin": {"position": [1, 9]},
"top_bin": {"position": [5, 9]},
},
}
def describe_state(world):
lines = [
f"Robot is at {world['robot']['position']}, gripper is {world['robot']['gripper']}.",
"Objects:",
]
for name, obj in world["objects"].items():
lines.append(f"- {name}: {obj['position']} ({obj['state']})")
lines.append("Bins:")
for name, bin_ in world["bins"].items():
lines.append(f"- {name}: {bin_['position']}")
return "\n".join(lines)
SYSTEM_PROMPT = """
You are the controller for a 2D pick-and-place robot.
The world is a 10 by 10 grid.
Valid actions, one per line:
- MOVE_TO x y
- GRIP
- RELEASE
Rules:
- You must MOVE_TO within 1 unit of an object before you GRIP it.
- After you GRIP, the object is attached to the gripper.
- MOVE_TO the destination bin before RELEASE.
- Do not explain. Output only the action list.
- If the command is impossible, output exactly: FAIL: reason
"""
def plan_actions(world, user_command):
state_text = describe_state(world)
user_message = f"""Current state:
{state_text}
User command: {user_command}
Actions:"""
response = client.chat.completions.create(
model="qwen-3-32b",
messages=[
{"role": "system", "content": SYSTEM_PROMPT},
{"role": "user", "content": user_message},
],
)
raw = response.choices[0].message.content.strip()
actions = [line.strip() for line in raw.splitlines() if line.strip()]
return actions
def execute_actions(world, actions):
robot = world["robot"]
for act in actions:
if act.startswith("FAIL:"):
print(f"Planner reported failure: {act}")
return
parts = act.split()
cmd = parts[0]
if cmd == "MOVE_TO":
x, y = int(parts[1]), int(parts[2])
if not (0 <= x <= world["bounds"][0] and 0 <= y <= world["bounds"][1]):
print(f"Invalid move: {x},{y}")
return
robot["position"] = [x, y]
print(f"Moved to {x},{y}")
elif cmd == "GRIP":
if robot["gripper"] != "empty":
print("Gripper already full")
return
held = None
for name, obj in world["objects"].items():
if obj["state"] != "on_table":
continue
dist = math.dist(robot["position"], obj["position"])
if dist <= 1.5:
held = name
break
if held is None:
print("No object in range to grip")
return
robot["gripper"] = held
world["objects"][held]["state"] = "in_gripper"
print(f"Gripped {held}")
elif cmd == "RELEASE":
if robot["gripper"] == "empty":
print("Nothing to release")
return
held = robot["gripper"]
robot["gripper"] = "empty"
world["objects"][held]["state"] = "in_bin"
world["objects"][held]["position"] = list(robot["position"])
print(f"Released {held} at {world['objects'][held]['position']}")
else:
print(f"Unknown action: {act}")
return
if __name__ == "__main__":
command = "pick up the blue cube and drop it in the top bin"
print(f"Command: {command}")
plan = plan_actions(WORLD, command)
print("\nPlanned actions:")
for a in plan:
print(a)
print("\nExecuting...")
execute_actions(WORLD, plan)
print("\nFinal state:")
print(describe_state(WORLD))Example output:
Command: pick up the blue cube and drop it in the top bin
Planned actions:
MOVE_TO 7 2
GRIP
MOVE_TO 5 9
RELEASE
Executing...
Moved to 7,2
Gripped blue_cube
Moved to 5,9
Released blue_cube at [5, 9]
Final state:
Robot is at [5, 9], gripper is empty.
Objects:
- red_box: [3, 3] (on_table)
- blue_cube: [5, 9] (in_bin)
Bins:
- left_bin: [1, 9]
- top_bin: [5, 9]Next steps
Swap the simulator for a real ROS2 node by publishing the same action strings to a robot topic. Or add a vision layer: feed camera frames to kimi-k2.6 on Oxlo.ai to generate the object list dynamically instead of hardcoding positions.
