§ 1.3 奥比中光 pyobbecsdk - 基础使用案例
1. 准备工作
在运行脚本之前，替换文件夹里面的动态链接库为自己编译的版本。
注意：动态链接库的Python版本需要与自己运行脚本的Python版本匹配才可以，不匹配不能正常使用
2. 导入依赖
import cv2
import numpy as np
import sys
from matplotlib import pyplot as plt
import open3d as o3d
# 奥比中光 SDK
from pyorbbecsdk import *
3. 配置参数
# 对齐模式设置为硬件对齐
# 硬件对齐由相机的芯片处理，不占用CPU的算力，推荐！
align_mode = "HW"
# 是否开启深度图与彩图同步
enable_sync = True
4. 创建Pipeline
# 创建Pipeline
pipeline = Pipeline()
config = Config()
5. 配置彩图视频流
# 获取彩图选项列表
color_profile_list = pipeline.get_stream_profile_list(OBSensorType.COLOR_SENSOR)
# 获取选项个数
profile_num = color_profile_list.get_count()
print(f"总共有{profile_num}个彩色图选项")
输出日志
[2023-08-12 17:11:31.745305][info][80468][Gemini2Device.cpp:662] Color sensor has been created! 总共有80个彩色图选项
# 获取默认的视频流配置
color_profile = color_profile_list.get_default_video_stream_profile()
# 配置使能彩图视频流
config.enable_stream(color_profile)
# 根据索引号获取特定profile的信息
# index = 0
# color_profile = color_profile_list.get_stream_profile_by_index(index)
# 打印视频流信息
profile_type = color_profile.get_type()
print(f"视频流类型: {profile_type}")
fmt = color_profile.get_format()
print(f"视频流格式: {fmt}")
width = color_profile.get_width()
height = color_profile.get_height()
fps =color_profile.get_fps()
print(f"分辨率 {width} x {height} 帧率 {fps}")
输出日志
视频流类型: OBStreamType.COLOR_STREAM 视频流格式: OBFormat.MJPG 分辨率 1280 x 720 帧率 30
6. 配置深度图视频流
# 获取深度图的选项列表
depth_profile_list = pipeline.get_stream_profile_list(OBSensorType.DEPTH_SENSOR)
# 获取选项个数
profile_num = depth_profile_list.get_count()
print(f"总共有{profile_num}个深度图选项")
输出日志
[2023-08-12 17:11:31.760036][info][80468][Gemini2Device.cpp:614] Depth sensor has been created! 总共有36个深度图选项
# 获取默认的视频流配置
depth_profile = depth_profile_list.get_default_video_stream_profile()
# 根据索引号获取特定profile的信息
# index = 0
# depth_profile = depth_profile_list.get_stream_profile_by_index(index)
# 配置使能深度图视频流
config.enable_stream(depth_profile)
# 打印视频流信息
profile_type = depth_profile.get_type()
print(f"视频流类型: {profile_type}")
fmt = depth_profile.get_format()
print(f"视频流格式: {fmt}")
width = depth_profile.get_width()
height = depth_profile.get_height()
fps =depth_profile.get_fps()
print(f"分辨率 {width} x {height} 帧率 {fps}")
输出日志
视频流类型: OBStreamType.DEPTH_STREAM 视频流格式: OBFormat.RLE 分辨率 1280 x 800 帧率 30
7. 设置对齐模式
if align_mode == 'HW':
config.set_align_mode(OBAlignMode.HW_MODE)
elif align_mode == 'SW':
config.set_align_mode(OBAlignMode.SW_MODE)
else:
config.set_align_mode(OBAlignMode.DISABLE)
8. 帧率同步
if enable_sync:
pipeline.enable_frame_sync()
9. 开启图像管道
pipeline.start(config)
输出日志
[2023-08-12 17:11:31.783490][info][80468][Pipeline.cpp:236] Try to start streams! [2023-08-12 17:11:31.783532][info][80468][VideoSensor.cpp:554] start OB_SENSOR_COLOR stream with profile: {type: OB_STREAM_COLOR, format: OB_FORMAT_MJPG, width: 1280, height: 720, fps: 30} [2023-08-12 17:11:31.788769][info][80468][VideoSensor.cpp:554] start OB_SENSOR_DEPTH stream with profile: {type: OB_STREAM_DEPTH, format: OB_FORMAT_RLE, width: 1280, height: 800, fps: 30} strmh->payload_header_hold_bytes is invalid,size:0[2023-08-12 17:11:31.827736][info][80468][Pipeline.cpp:249] Start streams done! [2023-08-12 17:11:31.827741][info][80468][Pipeline.cpp:232] Pipeline start done! strmh->payload_header_hold_bytes is invalid,size:0[2023-08-12 17:11:31.828823][warning][80498][FrameUnpacker.cpp:235] Current frame data size error after RLE Decode, size=46405, require=1792000 [2023-08-12 17:11:31.828857][warning][80498][FrameUnpacker.cpp:349] Frame Unpack failed due to original data is wrong! There may be interference in the device or data transmission.
10. 读取数据帧
def color_frame_to_bgr_img(frame):
'''将彩图数据帧转换为numpy格式的BGR彩图'''
width = frame.get_width()
height = frame.get_height()
color_format = frame.get_format()
data = np.asanyarray(frame.get_data())
image = np.zeros((height, width, 3), dtype=np.uint8)
if color_format == OBFormat.RGB:
image = np.resize(data, (height, width, 3))
image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
elif color_format == OBFormat.MJPG:
image = cv2.imdecode(data, cv2.IMREAD_COLOR)
else:
print("不支持彩图数据格式: {}".format(color_format))
return None
return image
def depth_frame_to_depth_img(frame):
'''深度数据帧转换为深度图'''
# 获取深度图宽度
width = depth_frame.get_width()
# 获取深度图高度
height = depth_frame.get_height()
# 获取深度图尺度
scale = depth_frame.get_depth_scale()
# 转换为numpy的float32格式的矩阵
depth_data = np.frombuffer(depth_frame.get_data(), dtype=np.uint16)
depth_data = depth_data.reshape((height, width))
depth_data = depth_data.astype(np.float32) * scale
return depth_data
# 跳过前几帧
for i in range(30):
pipeline.wait_for_frames(100)
color_img = None # 彩图
depth_img = None # 深度图
# 获取数据帧
frames = pipeline.wait_for_frames(100)
if frames is None:
print("[ERROR]数据帧获取失败")
else:
# 获取彩图数据帧
color_frame = frames.get_color_frame()
if color_frame is None:
print("[ERROR] 彩图获取失败")
else:
# 转换为OpenCV格式的彩图数据格式
color_img = color_frame_to_bgr_img(color_frame)
# 获取深度图数据帧
depth_frame = frames.get_depth_frame()
if depth_frame is None:
print("[ERROR] 深度图获取失败")
else:
depth_img = depth_frame_to_depth_img(depth_frame)
# 可视化彩图
plt.imshow(color_img[:,:,::-1])
# 可视化深度图
plt.imshow(depth_img)
# 保存彩图
cv2.imwrite("color_img.png", color_img)
# 保存深度图
np.save("depth_img.npy", depth_img)
11. 获取相机内参
camera_param = pipeline.get_camera_param()
print("彩色相机内参")
print(camera_param.rgb_intrinsic)
print(f"彩色相机畸变系数")
print(camera_param.rgb_distortion)
print(f"深度图相机内参")
print(camera_param.depth_intrinsic)
print(f"深度图相机畸变系数")
print(camera_param.depth_distortion)
print("深度相机到彩色相机坐标系的空间变换")
print(camera_param.transform)
输出日志
彩色相机内参 <OBCameraIntrinsic fx=690.472290 fy=690.000366 cx=639.147827 cy=360.172974 width=1280 height=720> 彩色相机畸变系数 <OBCameraDistortion k1=51.501217 k2=-103.048347 k3=119.678688 k4=51.351246 k5=-102.766083 k6=120.118965 p1=0.000422 p2=0.000248> 深度图相机内参 <OBCameraIntrinsic fx=690.472290 fy=690.000366 cx=639.147827 cy=360.172974 width=1280 height=720> 深度图相机畸变系数 <OBCameraDistortion k1=0.000000 k2=0.000000 k3=0.000000 k4=0.000000 k5=0.000000 k6=0.000000 p1=0.000000 p2=0.000000> 深度相机到彩色相机坐标系的空间变换 <OBD2CTransform rot=[0.99987, -0.0158098, -0.00326567, 0.0158227, 0.999867, 0.00394567, 0.00320285, -0.00399683, 0.999987] transform=[-14.1013, -0.307893, -1.933]
在实际使用的过程中，只会用到彩图相机内参。获取的彩图，实际上是移除了畸变之后的。
# 获取彩色相机内参
fx = camera_param.rgb_intrinsic.fx
fy = camera_param.rgb_intrinsic.fy
cx = camera_param.rgb_intrinsic.cx
cy = camera_param.rgb_intrinsic.cy
# 彩图尺寸
img_width = camera_param.rgb_intrinsic.width
img_height = camera_param.rgb_intrinsic.height
# 构造内参矩阵
intrinsic = np.float32([
[fx, 0, cx],
[0, fy, cy],
[0, 0, 1]])
print("彩色相机的内参矩阵:")
print(intrinsic)
输出日志
彩色相机的内参矩阵: [[690.4723 0. 639.1478 ] [ 0. 690.00037 360.17297] [ 0. 0. 1. ]]
# 生成Open3D的内参格式
pinhole_camera_intrinsic = o3d.camera.PinholeCameraIntrinsic(
img_width, img_height, \
fx, fy, cx, cy)
12. 转换为Open3D格式的彩色点云
def create_point_cloud(color_image, depth_image, camera_param, depth_scale=1000.0):
'''创建点云(向量化操作)
@color_image: 彩图
@depth_image: 深度图
@depth_scale: 深度图单位/尺度
一般深度图单位是mm， 转换为m需要/1000.0
'''
# 获取彩色相机内参
fx = camera_param.rgb_intrinsic.fx
fy = camera_param.rgb_intrinsic.fy
cx = camera_param.rgb_intrinsic.cx
cy = camera_param.rgb_intrinsic.cy
# 彩图尺寸
img_width = camera_param.rgb_intrinsic.width
img_height = camera_param.rgb_intrinsic.height
# 缩放深度图
if depth_scale != 1.0:
depth_image = depth_image / depth_scale
# 得到索引号
valid_index = depth_image != 0
# 得到有效点云的RGB数值
color_rgb = cv2.cvtColor(color_image, cv2.COLOR_BGR2RGB)
colors = color_rgb[valid_index].reshape((-1, 3)) / 255
# 创建一个空的点云对象
pcd = o3d.geometry.PointCloud()
# 根据相机内参矩阵计算3D坐标
py, px = np.indices((img_height, img_width))
# 提取
px_valid = px[valid_index]
py_valid = py[valid_index]
z = depth_image[valid_index]
# 计算相机坐标系下的三维坐标
x = (px_valid - cx) * z / fx
y = (py_valid - cy) * z / fy
points = np.stack([x, y, z], axis=-1)
# 将3D坐标转换为点云对象
points = points.reshape(-1, 3)
# 填充PCD对象
pcd.points = o3d.utility.Vector3dVector(points)
pcd.colors = o3d.utility.Vector3dVector(colors)
return pcd
# 获取Open3D PCD格式的点云
pcd = create_point_cloud(color_img, depth_img, camera_param)
# 点云可视化
o3d.visualization.draw_geometries([pcd])
13. 视频流使能
# 关闭所有的视频流
config.disable_all_stream()
# 使能所有的视频流
# config.enable_all_stream()
14. 【不推荐】获取奥比中光的点云类 (XYZ)
注意：不建议使用奥比中光的点云对象。用Open3D的点云格式更好
def save_points2ply(points, ply_path):
'''将点云对象保存为PLY点云格式'''
with open(ply_path, "w") as f:
f.write("ply\n")
f.write("format ascii 1.0\n")
f.write("element vertex {}\n".format(len(points)))
f.write("property float x\n")
f.write("property float y\n")
f.write("property float z\n")
f.write("end_header\n")
for point in points:
f.write("{} {} {}\n".format(point.x, point.y, point.z))
# 转换为点云
points = frames.convert_to_points(camera_param)
print(f"点的个数: {len(points)}")
输出日志
点的个数: 915680
point_index = 0 # 点的索引号
point = points[point_index]
print(f"点的坐标 X={point.x} Y={point.y} Z={point.z}")
输出日志
点的坐标 X=-257.3915710449219 Y=148.76968383789062 Z=285.20001220703125
# 保存点云
# 注: 字符串格式 速度很慢
ply_path = "points.ply"
save_points2ply(points, ply_path)
15. 【不推荐】获取奥比中光的彩色点云类
def save_points2ply_color(points, ply_path):
'''将点云对象保存为PLY彩色点云格式'''
with open(ply_path, "w") as f:
f.write("ply\n")
f.write("format ascii 1.0\n")
f.write("element vertex {}\n".format(len(points)))
f.write("property float x\n")
f.write("property float y\n")
f.write("property float z\n")
f.write("property uchar red\n")
f.write("property uchar green\n")
f.write("property uchar blue\n")
f.write("end_header\n")
for point in points:
f.write(
"{} {} {} {} {} {}\n".format(point.x, point.y, point.z, point.r, point.g, point.b))
注意：不建议使用奥比中光的点云对象。用Open3D的点云格式更好
# 转换为彩色点云
color_points = frames.convert_to_color_points(camera_param)
print(f"点的个数: {len(points)}")
输出日志
点的个数: 915680
point_index = 0 # 点的索引号
point = color_points[point_index]
print(f"点的坐标 X={point.x} Y={point.y} Z={point.z}")
print(f"点的颜色 R={point.r} G={point.g} B={point.b}")
输出日志
点的坐标 X=-257.3915710449219 Y=148.76968383789062 Z=285.20001220703125 点的颜色 R=24 G=9 B=16
保存点云为ply格式
# 保存点云
# 注: 字符串格式 速度很慢
ply_path = "./color_points.ply"
save_points2ply_color(color_points, ply_path)