Readme and comments

Author: Yuxiang Gao

README.md

@@ -0,0 +1,18 @@
+# PATI
+## Introduction
+This is the repository accompanying the paper *PATI: A Projection-Based Augmented Table-Top Interface for Robot Programming*. In this work, we present PATI—a Projection-based Augmented Table-top Interface for robot programming—through which users are able to use sim- ple, common gestures (e.g., pinch gestures) and tools (e.g., shape tools) to specify table-top manipulation tasks (such as pick-and-place) for a robot manipulator. PATI allows users to interact with the environment directly when providing task specifications.
+
+## Usage
+`roslaunch ropi_tangible_surface all_in_one.launch`
+`rosrun ropi_tangible_surface tangible_surface.py`
+
+## Structure
+- ropi_msgs: ROS messages, services and actions
+- ropi_tangible_surface: main package
+    - ropi_tangible_surface: python package for backend processing
+        - base_class.py: base class of all detection and tracking clsses.
+        - finger_detection.py: detect fingertip positions
+        - fingertip_tracking.py: filter and track detections
+        - object_detection.py: detect tabletop objects
+        - object_tracking.py: filter and track object detections
+    - tangible_surface.py: main script

ropi_tangible_surface/scripts/ropi_tangible_surface/fingertip_tracking.py

@@ -49,7 +49,6 @@ def next_valid(self, id):
 class TouchTracker(TrackerBase):
     def __init__(self, pt, id=None):
         super(TouchTracker, self).__init__()
-        # self.id = id
         self.position = pt
         self.position_prev = pt
         self.last_time = 0
@@ -60,6 +59,7 @@ def __init__(self, pt, id=None):
         self.smoothing_factor = 0
 
     def elapsed_time(self):
+        '''Calculate elapsed time'''
         if self.last_time != 0:
             elapsed = rospy.get_time() - self.time
             # self.last_time = rospy.get_time()
@@ -68,6 +68,8 @@ def elapsed_time(self):
             return 0
 
     def update(self, pos):
+        '''Make an update
+        input: fingertip position'''
         if pos is not None:
             # clear release cnt
             self.release_cnt = 0
@@ -121,6 +123,8 @@ def __init__(self, screen_shape):
 
 
     def update(self, pts):
+        '''Make an update
+        input: list of fingertip positions'''
         if DEBUG: print('update', pts)
         if len(self.trackers) == 0:
             self.new_trackers(pts)
@@ -129,13 +133,15 @@ def update(self, pts):
             # print(self.trackers)
 
     def new_trackers(self, pts):
+        '''init new tracker'''
         for pt in pts:
             if not ma.is_masked(pt):
                 new_cursor = TouchTracker(pt)
                 self.trackers.append(new_cursor)
                 if DEBUG: print('add pt', new_cursor.id.hex)
 
     def match_trackers(self, pts):
+        '''search and match existing points with incoming points'''
         # when only one finger, allow more tolerence
         if len(pts) == 1:
             move_threshold = 65
@@ -169,12 +175,13 @@ def match_trackers(self, pts):
                     self.release_tracker(cur)
 
     def release_tracker(self, cur):
+        '''call release tracker service '''
         try:
             print('---------Cursor release-----------')
             self.delete_cursor = rospy.ServiceProxy('delete_cursor', DeleteSelection)
             self.delete_cursor(cur.id.hex)
-        except rospy.ServiceException, e:
-            print "Service call failed: %s"%e
+        except rospy.ServiceException as e:
+            print("Service call failed: %s"%e)
         cur.state = CursorState['RELEASED']
         self.trackers.remove(cur)
         # if self.id_manager.release_id(cur.id):
@@ -183,6 +190,7 @@ def release_tracker(self, cur):
         #     if DEBUG: print('Cursor id release unsuccessful!!')
 
     def make_msg(self):
+        '''compile ros msg for communication'''
         msg = MultiTouch()
         msg.width = self.width
         msg.height = self.height

ropi_tangible_surface/scripts/ropi_tangible_surface/object_detection.py

@@ -9,6 +9,7 @@
 from geometry_msgs.msg import Point
 
 class GraspDataClass:
+    '''container class for objects' information'''
     def __init__(self, position, diameter, angle, height):
         self.position = np.asarray(position)
         self.target_position = None
@@ -62,6 +63,11 @@ def set_debug_img(self, img):
             self.debug_img = img
 
     def update(self, cnts, depth_img, debug_img=None):
+        '''update objects
+        input: 
+            cnts: list of contours
+            depth_img: depth image for detection
+            debug_img(optional): rgb img for visualization'''
         self.init()
         output = []
         if self.debug and debug_img is None:
@@ -94,6 +100,7 @@ def draw_selections(self, img, rects):
         return debug_img
 
 class ObjectDetection:
+    '''object detection class'''
     def __init__(self, debug=True):
         self.clear()
         self.debug = debug
@@ -124,6 +131,13 @@ def clear(self):
         self.debug_img = None
 
     def update(self, cnt, depth_img, debug_img=None):
+        '''   
+        make update
+        inputs:
+            cnt: contour of object
+            depth_img: depth image for detection
+            debug_img(optional): rgb img for visualization
+        '''
         self.clear()
         if self.debug and debug_img is None:
             self.set_debug_img(depth_img)

ropi_tangible_surface/scripts/tangible_surface.py

@@ -48,6 +48,7 @@ def load_data(self):
         self.depth_background = np.load(self.root_path + '/config/depth.npy')
 
     def on_init(self):
+        '''init all modules'''
         rospy.loginfo("on init")
         # Instances
         self.bridge = CvBridge()
@@ -96,7 +97,7 @@ def detect_objects_in_region(self, msg):
         return grasp_points
 
     def move_objects_callback(self, req):
-        # TODO: finish this
+        '''callback from move object request'''
         rospy.loginfo("move objects service called.")
         response = MoveObjectsResponse()
         if not self.is_moving:
@@ -135,6 +136,7 @@ def place_position(self, pick_pos, source, target):
         return target.normalized_rect.get_center() + relative_pos
 
     def mission_from_regions(self, source_region, target_region):
+        '''generate mission from source and target regions'''
         print('Source: ', source_region)
         print('Target: ', target_region)
         grasp_points = self.detect_objects_in_region(source_region)
@@ -155,6 +157,7 @@ def mission_from_regions(self, source_region, target_region):
             return None
 
     def delete_selection_callback(self, req):
+        '''callback function to delete a selection service'''
         rospy.loginfo("Delete selection service called.")
         print (self.selection_manager.selections)
         self.selection_manager.delete([req.guid])
@@ -186,6 +189,7 @@ def region_selection_callback(self, req):
         return response
 
     def image_callback(self, depth_in, rgb_in):
+        '''image preprocessing'''
         cv_rgb = self.rgb_callback(rgb_in)
         cv_depth = self.depth_callback(depth_in)
         depth_foreground = self.depth_background - cv_depth
@@ -270,6 +274,7 @@ def make_mask(self, pts, shape):
         return mask
 
     def detect_fingertip(self, img):
+        '''finger tip detection and tracking pipeline'''
         threshed = my_threshold(img, 0, 500)
         mask = np.zeros(img.shape, dtype=np.uint8)
         mask[threshed > 0] = 255
@@ -308,6 +313,7 @@ def detect_fingertip(self, img):
         return p
 
     def detect_object(self, img):
+        '''object detection and tracking pipeline'''
         objects = []
         threshed = my_threshold(img, 5, 150)
         mask = np.zeros(img.shape, dtype=np.uint8)

ur5_gripper_control/README.md

@@ -1 +1 @@
-# ur3_gripper85
+# ur5_gripper145