diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..9b0d3a7 --- /dev/null +++ b/.flake8 @@ -0,0 +1,6 @@ +[flake8] +extend-ignore = B902,C816,D100,D101,D102,D103,D104,D105,D106,D107,D203,D212,D404,I202,Q000 +import-order-style = google +max-line-length = 125 +show-source = true +statistics = true \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index 297ee22..830521c 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -13,4 +13,4 @@ about: "For feature requests. Please search for existing issues first. Also see * Tradeoffs made in design decisions * Caveats and considerations for the future -If there are multiple solutions, please present each one separately. Save comparisons for the very end.) \ No newline at end of file +If there are multiple solutions, please present each one separately. Save comparisons for the very end.) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml new file mode 100644 index 0000000..8bea564 --- /dev/null +++ b/.github/workflows/build_and_test.yml @@ -0,0 +1,65 @@ +name: Build and Test Workflows + +on: + pull_request: + branches: + - main + - develop + push: + branches: + - main + +jobs: + ament_lint: + name: Lint and Codecheck + runs-on: ubuntu-latest + steps: + - name: Code Checkout + uses: actions/checkout@v4 + - name: Setup ROS + uses: ros-tooling/setup-ros@master + - name: Ament Copyright Lint + uses: ros-tooling/action-ros-lint@master + with: + distribution: humble + linter: copyright + package-name: "*" + - name: Ament Flake8 Lint + uses: ros-tooling/action-ros-lint@master + with: + distribution: humble + linter: flake8 + package-name: "*" + arguments: '--config=${{ github.workspace }}/.flake8' + - name: Ament PEP257 Lint + uses: ros-tooling/action-ros-lint@master + with: + distribution: humble + linter: pep257 + package-name: "*" + - name: Ament xmllint Lint + uses: ros-tooling/action-ros-lint@master + with: + distribution: humble + linter: xmllint + package-name: "*" + + build_source: + name: Build Docker Image and ROS 2 Packages + runs-on: ubuntu-latest + env: + CONTAINER_NAME: perception_pipeline + PERCEP_WS: /root/percep_ws + steps: + - name: Code Checkout + uses: actions/checkout@v4 + - name: Build Docker Image + run: docker build . --file Dockerfile.ubuntu -t ${{ github.repository }}:latest + - name: Run Docker Image + run: | + docker run -it -d --name $CONTAINER_NAME \ + -v ${{ github.workspace }}:$PERCEP_WS/src \ + ${{ github.repository }}:latest + - name: Build ROS 2 Packages in Container + run: docker exec $CONTAINER_NAME bash -c \ + "source /opt/ros/humble/setup.bash && cd $PERCEP_WS && colcon build" diff --git a/.gitignore b/.gitignore index 684bac1..9f2ae87 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,82 @@ -#Ignore the test node -/object_detection/object_detection/test_node.py +# Custom settings asset +*.settings.asset* -#Ignore pycache dirs -object_detection/object_detection/Detectors/__pycache__/ -object_detection/object_detection/__pycache__/ +# Visual Studio 2015 cache directory +/Project/.vs/ -#Ignore .vscode dir -.vscode \ No newline at end of file +/[Ll]ibrary/ +/[Tt]emp/ +/[Oo]bj/ +/[Bb]uild/ +/[Bb]uilds/ +/[Ll]ogs/ +/[Mm]emoryCaptures/ + +# Autogenerated VS/MD/Consulo solution and project files +*.csproj +*.unityproj +*.sln +*.suo +*.tmp +*.user +*.userprefs +*.pidb +*.booproj +*.svd +*.pdb + +# Unity3D generated meta files +*.pidb.meta + +# Builds +*.apk +*.unitypackage +*.app +*.exe +*.x86_64 +*.x86 + +# Generated doc folders +/docs/html + +# Mac hidden files +*.DS_Store +*/.ipynb_checkpoints +*/.idea +*.pyc +*.idea/misc.xml +*.idea/modules.xml +*.idea/ +*.iml +*.cache +*/build/ +*/dist/ +*.egg-info* +*.eggs* +*.gitignore.swp + +# VSCode hidden files +.vscode/ + +.DS_Store +.ipynb_checkpoints + +# pytest cache +*.pytest_cache/ + +# Ignore PyPi build files. +dist/ +build/ + +# Python virtual environment +venv/ +.mypy_cache/ + +# Code coverage report +.coverage +coverage.xml +/htmlcov/ + +**/UserSettings/* + +ROSConnectionPrefab.prefab \ No newline at end of file diff --git a/Dockerfile.cuda b/Dockerfile.cuda new file mode 100644 index 0000000..f784bd2 --- /dev/null +++ b/Dockerfile.cuda @@ -0,0 +1,80 @@ +# Use cuda_version arg to take CUDA version as input from user +ARG cuda_version=11.8.0 + +# Use NVIDA-CUDA's base image +FROM nvcr.io/nvidia/cuda:${cuda_version}-devel-ubuntu22.04 + +# Prevent console from interacting with the user +ARG DEBIAN_FRONTEND=noninteractive + +# Prevent hash mismatch error for apt-get update, qqq makes the terminal quiet while downloading pkgs +RUN apt-get clean && rm -rf /var/lib/apt/lists/* && apt-get update -yqqq + +# Set folder for RUNTIME_DIR. Only to prevent warnings when running RViz2 and Gz +RUN mkdir tmp/runtime-root && chmod 0700 tmp/runtime-root +ENV XDG_RUNTIME_DIR='/tmp/runtime-root' + +RUN apt-get update + +RUN apt-get install --no-install-recommends -yqqq \ + apt-utils \ + nano \ + git + +# Using shell to use bash commands like 'source' +SHELL ["/bin/bash", "-c"] + +# Python Dependencies +RUN apt-get install --no-install-recommends -yqqq \ + python3-pip + +# Add locale +RUN locale && \ + apt update && apt install --no-install-recommends -yqqq locales && \ + locale-gen en_US en_US.UTF-8 && \ + update-locale LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 && \ + export LANG=en_US.UTF-8 && \ + locale + +# Setup the sources +RUN apt-get update && apt-get install --no-install-recommends -yqqq software-properties-common curl && \ + add-apt-repository universe && \ + curl -sSL https://raw.githubusercontent.com/ros/rosdistro/master/ros.key -o /usr/share/keyrings/ros-archive-keyring.gpg && \ + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/ros-archive-keyring.gpg] http://packages.ros.org/ros2/ubuntu $(. /etc/os-release && echo $UBUNTU_CODENAME) main" | tee /etc/apt/sources.list.d/ros2.list > /dev/nul + +# Install ROS 2 Humble +RUN apt update && apt install --no-install-recommends -yqqq \ + ros-humble-ros-base \ + ros-dev-tools + +# Install cv-bridge +RUN apt update && apt install --no-install-recommends -yqqq \ + ros-humble-cv-bridge + +# Target workspace for ROS2 packages +ARG WORKSPACE=/root/percep_ws + +# Add target workspace in environment +ENV WORKSPACE=$WORKSPACE + +# Creating the models folder +RUN mkdir -p $WORKSPACE/models && \ + mkdir -p $WORKSPACE/src + +# Installing Python dependencies +COPY object_detection/requirements.txt . +RUN pip3 install -r requirements.txt + +# ROS Dependencies +RUN apt update && apt install --no-install-recommends -yqqq \ + ros-humble-cyclonedds \ + ros-humble-rmw-cyclonedds-cpp + +# Use cyclone DDS by default +ENV RMW_IMPLEMENTATION rmw_cyclonedds_cpp + +WORKDIR /root/percep_ws + +# Update .bashrc +RUN echo "source /opt/ros/humble/setup.bash" >> /root/.bashrc + diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu new file mode 100644 index 0000000..2a05f69 --- /dev/null +++ b/Dockerfile.ubuntu @@ -0,0 +1,77 @@ +# Use Ubuntu's base image +FROM ubuntu:22.04 + +# Prevent console from interacting with the user +ARG DEBIAN_FRONTEND=noninteractive + +# Prevent hash mismatch error for apt-get update, qqq makes the terminal quiet while downloading pkgs +RUN apt-get clean && rm -rf /var/lib/apt/lists/* && apt-get update -yqqq + +# Set folder for RUNTIME_DIR. Only to prevent warnings when running RViz2 and Gz +RUN mkdir tmp/runtime-root && chmod 0700 tmp/runtime-root +ENV XDG_RUNTIME_DIR='/tmp/runtime-root' + +RUN apt-get update + +RUN apt-get install --no-install-recommends -yqqq \ + apt-utils \ + nano \ + git + +# Using shell to use bash commands like 'source' +SHELL ["/bin/bash", "-c"] + +# Python Dependencies +RUN apt-get install --no-install-recommends -yqqq \ + python3-pip + +# Add locale +RUN locale && \ + apt update && apt install --no-install-recommends -yqqq locales && \ + locale-gen en_US en_US.UTF-8 && \ + update-locale LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 && \ + export LANG=en_US.UTF-8 && \ + locale + +# Setup the sources +RUN apt-get update && apt-get install --no-install-recommends -yqqq software-properties-common curl && \ + add-apt-repository universe && \ + curl -sSL https://raw.githubusercontent.com/ros/rosdistro/master/ros.key -o /usr/share/keyrings/ros-archive-keyring.gpg && \ + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/ros-archive-keyring.gpg] http://packages.ros.org/ros2/ubuntu $(. /etc/os-release && echo $UBUNTU_CODENAME) main" | tee /etc/apt/sources.list.d/ros2.list > /dev/nul + +# Install ROS 2 Humble +RUN apt update && apt install --no-install-recommends -yqqq \ + ros-humble-ros-base \ + ros-dev-tools + +# Install cv-bridge +RUN apt update && apt install --no-install-recommends -yqqq \ + ros-humble-cv-bridge + +# Target workspace for ROS2 packages +ARG WORKSPACE=/root/percep_ws + +# Add target workspace in environment +ENV WORKSPACE=$WORKSPACE + +# Creating the models folder +RUN mkdir -p $WORKSPACE/models && \ + mkdir -p $WORKSPACE/src + +# Installing Python dependencies +COPY object_detection/requirements.txt . +RUN pip3 install -r requirements.txt + +# ROS Dependencies +RUN apt update && apt install --no-install-recommends -yqqq \ + ros-humble-cyclonedds \ + ros-humble-rmw-cyclonedds-cpp + +# Use cyclone DDS by default +ENV RMW_IMPLEMENTATION rmw_cyclonedds_cpp + +WORKDIR /root/percep_ws + +# Update .bashrc +RUN echo "source /opt/ros/humble/setup.bash" >> /root/.bashrc + diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..919de01 --- /dev/null +++ b/LICENSE @@ -0,0 +1,195 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + Copyright (c)2023 A.T.O.M ROBOTICS + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + + + \ No newline at end of file diff --git a/README.md b/README.md index 49abc22..448d24a 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,19 @@ These components can be stitched together to make a custom pipeline for any use- Follow these steps to setup this project on your systm ### Prerequisites +Install the binary Gazebo Garden/ROS 2 Humble packages: + +* Follow [these](https://gazebosim.org/docs/garden/install_ubuntu#binary-installation-on-ubuntu) instructions to install gz-garden from packages.osrfoundation.org repository. +* Install ros_gz + * From the non official binary packages from apt: + + * ```apt-get install ros-humble-ros-gzgarden``` + * Build from source: + * Refer to the [official ros_gz repository](https://github.com/gazebosim/ros_gz/tree/humble#from-source) + +Install docker and add it to user group: + +* Refer to this [link](https://cloudyuga.guru/hands_on_lab/docker-as-non-root-user) Follow these steps to install ROS Humble and OpenCV * ROS Humble @@ -116,33 +129,54 @@ Refer to the official [ROS 2 installation guide](https://docs.ros.org/en/humble/ ### Installation -1. Make a new workspace +1. **Run using Docker** + ```bash - mkdir -p percep_ws/src + cd docker_scripts + export PERCEP_WS_PATH= + ./run_devel.sh ``` -2. Clone the ROS-Perception-Pipeline repository - - Now go ahead and clone this repository inside the "src" folder of the workspace you just created. - - ```bash - cd percep_ws/src - git clone git@github.com:atom-robotics-lab/ros-perception-pipeline.git - ``` - -3. Compile the package - - Follow this execution to compile your ROS 2 package - - ```bash - colcon build --symlink-install - ``` - -4. Source your workspace - ```bash - source install/local_setup.bash - ``` - +2. **Run natively** + + 1. Make a new workspace + ```bash + mkdir -p percep_ws/src + ``` + + 2. Clone the ROS-Perception-Pipeline repository + + Now go ahead and clone this repository inside the "src" folder of the workspace you just created. + + ```bash + cd percep_ws/src && git clone git@github.com:atom-robotics-lab/ros-perception-pipeline.git + ``` + 3. Install dependencies using rosdep + + Update Your rosdep before installation. + + ```bash + rosdep update + ``` + + This command installs all the packages that the packages in your catkin workspace depend upon but are missing on your computer. + ```bash + rosdep install --from-paths src --ignore-src -r -y + ``` + + 4. Compile the package + + Follow this execution to compile your ROS 2 package + + ```bash + colcon build --symlink-install + ``` + + 5. Source your workspace + + ```bash + source install/local_setup.bash + ``` @@ -166,7 +200,7 @@ Don't forget to click on the **play** button on the bottom left corner of the Ig
-### 2. Launch the Object Detection node +### 2.1 Launch the Object Detection node natively
Use the pip install command as shown below to install the required packages. @@ -178,11 +212,38 @@ Use the command given below to run the ObjectDetection node. Remember to change file according to your present working directory ```bash -ros2 run object_detection ObjectDetection --ros-args --params-file src/ros-perception-pipeline/object_detection/config/object_detection.yaml +ros2 launch object_detection object_detection.launch.py ``` +**Note :** If your imports are not working while using a virtual environment, you'll need to manually set your `PYTHONPATH` environment variable. +Follow these steps to do this : + +1. Activate your virtual environment + +2. Find out the path of your virtual environment's Python installation + ```bash + which Python + ``` + +3. Set your `PYTHONPATH` + ```bash + export PYTHONPATH = {insert_your_python_path_here} + ``` + +### 2.2 Launch the Object Detection node using Docker +
+ +We can use the Docker image built previously to run the `object_detection` package + + ```bash + colcon build --symlink-install + source install.setup.bash + ros2 launch object_detection object_detection.launch.py + ``` + ### 3. Changing the Detector + To change the object detector being used, you can change the parameters inside the object_detection.yaml file location inside the **config** folder. @@ -199,8 +260,7 @@ ros2 run rqt_image_view rqt_image_view
-

(back to top)

- +

(back to top)

+ + diff --git a/docker_scripts/run_devel.sh b/docker_scripts/run_devel.sh new file mode 100755 index 0000000..a29ce24 --- /dev/null +++ b/docker_scripts/run_devel.sh @@ -0,0 +1,72 @@ +#!/bin/bash + +xhost +local:root + +IMAGE_NAME="perception_pipeline" +IMAGE_TAG="latest" +CONTAINER_NAME="perception_pipeline" + +# Build the image if it doesn't exist +if docker inspect "$IMAGE_NAME:$IMAGE_TAG" &> /dev/null; then + echo "The image $IMAGE_NAME:$IMAGE_TAG exists." + +else + echo "The image $IMAGE_NAME:$IMAGE_TAG does not exist. Building the image...." + + echo "Choose the base image:" + echo "1. NVIDIA CUDA image" + echo "2. Ubuntu 22.04 image" + read -p "Enter your choice (1 or 2): " base_image_choice + + # If the user input is blank or not 1 or 2, default to NVIDIA CUDA image + if [ -z "$base_image_choice" ] || [ "$base_image_choice" != "1" ] && [ "$base_image_choice" != "2" ]; then + base_image_choice="1" + fi + + # Choose the appropriate Dockerfile based on user input + if [ "$base_image_choice" == "1" ]; then + DOCKERFILE="Dockerfile.cuda" + + echo "Enter your preferred CUDA Version (default set to 11.8.0) : " + read cuda_version + + # If the user input is blank, use 11.8.0 as the cuda_version + if [ -z "$cuda_version" ]; then + cuda_version="11.8.0" + fi + + cd .. + docker build --build-arg cuda_version="$cuda_version" -f "$DOCKERFILE" -t "$IMAGE_NAME:$IMAGE_TAG" . + echo "Completed building the docker image" + else + DOCKERFILE="Dockerfile.ubuntu" + + cd .. + docker build -f "$DOCKERFILE" -t "$IMAGE_NAME:$IMAGE_TAG" . + fi +fi + +# Enter into the container if it is already running +if [ "$(docker ps -a --quiet --filter status=running --filter name=$CONTAINER_NAME)" ]; then + echo -e "\nAttaching to running container: $CONTAINER_NAME" + docker exec -it $CONTAINER_NAME /bin/bash $@ + exit 0 +fi + +# Check if the PERCEP_WS_PATH environment variable is empty +if [ -z "$PERCEP_WS_PATH" ]; then + echo -e "\nThe environment variable : PERCEP_WS_PATH is empty. Point it to the path of the ROS 2 workspace in which the ros-perception-pipeline project is kept !!" + exit 1 +fi + +# Run the docker container +docker run --gpus all --shm-size=1g --ulimit memlock=-1 --ulimit stack=67108864 \ +-it --rm --privileged --net=host --ipc=host \ +--name $CONTAINER_NAME \ +-v $PERCEP_WS_PATH/src/:/root/percep_ws/src \ +-v $PERCEP_WS_PATH/models/:/root/percep_ws/models/ \ +-v ddsconfig.xml:/ddsconfig.xml \ +--env CYCLONEDDS_URI=/ddsconfig.xml \ +--env="QT_X11_NO_MITSHM=1" \ +--env="DISPLAY" \ +$IMAGE_NAME:$IMAGE_TAG diff --git a/object_detection/config/params.yaml b/object_detection/config/params.yaml index 909a042..681c2b9 100644 --- a/object_detection/config/params.yaml +++ b/object_detection/config/params.yaml @@ -1,11 +1,12 @@ object_detection: ros__parameters: - input_img_topic: color_camera/image_raw + input_img_topic: /kitti/camera/color/left/image_raw output_bb_topic: object_detection/img_bb output_img_topic: object_detection/img + publish_output_img: 1 model_params: detector_type: YOLOv5 - model_dir_path: models/ - weight_file_name: auto_final.onnx - confidence_threshold : 0.7 + model_dir_path: /root/percep_ws/models/yolov5 + weight_file_name: yolov5s.pt + confidence_threshold : 0.5 show_fps : 1 \ No newline at end of file diff --git a/object_detection/launch/object_detection.launch.py b/object_detection/launch/object_detection.launch.py index f1f1980..65804df 100644 --- a/object_detection/launch/object_detection.launch.py +++ b/object_detection/launch/object_detection.launch.py @@ -1,10 +1,10 @@ -# Copyright (c) 2018 Intel Corporation +# Copyright (c) 2023 A.T.O.M ROBOTICS # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -13,14 +13,9 @@ # limitations under the License. import os -import sys from ament_index_python.packages import get_package_share_directory - from launch import LaunchDescription -from launch.actions import IncludeLaunchDescription, DeclareLaunchArgument -from launch.substitutions import LaunchConfiguration -from launch.launch_description_sources import PythonLaunchDescriptionSource from launch_ros.actions import Node @@ -32,14 +27,14 @@ def generate_launch_description(): 'config', 'params.yaml' ) - - node=Node( - package = 'object_detection', - name = 'object_detection', - executable = 'ObjectDetection', - parameters = [params], - output="screen" + + node = Node( + package='object_detection', + name='object_detection', + executable='ObjectDetection', + parameters=[params], + output="screen", + emulate_tty=True ) - return LaunchDescription([node]) diff --git a/object_detection/object_detection/DetectorBase.py b/object_detection/object_detection/DetectorBase.py index 801b5cc..d0b11f5 100644 --- a/object_detection/object_detection/DetectorBase.py +++ b/object_detection/object_detection/DetectorBase.py @@ -1,4 +1,19 @@ +# Copyright (c) 2023 A.T.O.M ROBOTICS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from abc import ABC, abstractmethod + import numpy as np @@ -8,6 +23,7 @@ def __init__(self) -> None: self.predictions = [] def create_predictions_list(self, class_ids, confidences, boxes): + self.predictions = [] for i in range(len(class_ids)): obj_dict = { "class_id": class_ids[i], @@ -16,7 +32,7 @@ def create_predictions_list(self, class_ids, confidences, boxes): } self.predictions.append(obj_dict) - + @abstractmethod def build_model(self, model_dir_path: str, weight_file_name: str) -> None: pass @@ -27,4 +43,4 @@ def load_classes(self, model_dir_path: str) -> None: @abstractmethod def get_predictions(self, cv_image: np.ndarray) -> list[dict]: - pass \ No newline at end of file + pass diff --git a/object_detection/object_detection/Detectors/EfficientDet.py b/object_detection/object_detection/Detectors/EfficientDet.py deleted file mode 100644 index e5a5f49..0000000 --- a/object_detection/object_detection/Detectors/EfficientDet.py +++ /dev/null @@ -1,196 +0,0 @@ -import tensorflow_hub as hub -import cv2 -import numpy -import pandas as pd -import tensorflow as tf -import matplotlib.pyplot as plt - -import tempfile - - -# For drawing onto the image. -import numpy as np -from PIL import Image -from PIL import ImageColor -from PIL import ImageDraw -from PIL import ImageFont -from PIL import ImageOps - -# For measuring the inference time. -import time - -class EfficientDet: - def __init__(self, model_dir_path, weight_file_name, conf_threshold = 0.7, score_threshold = 0.25, nms_threshold = 0.4): - - self.frame_count = 0 - self.total_frames = 0 - self.fps = -1 - self.start = time.time_ns() - self.frame = None - - self.model_dir_path = model_dir_path - self.weight_file_name = weight_file_name - self.conf=conf_threshold - - # Resizing image - self.img_height=800 - self.img_width=800 - self.predictions=[] - - self.build_model() - self.load_classes() - - def build_model(self) : - module_handle="https://tfhub.dev/tensorflow/efficientdet/d0/1" - # Loading model directly from TensorFlow Hub - self.detector = hub.load(module_handle) - - - def load_classes(self): - self.labels = [] - with open(self.model_dir_path + "/classes.txt", "r") as f: - self.labels = [cname.strip() for cname in f.readlines()] - return self.labels - - def display_image(self,image): - cv2.imshow("result", image) - cv2.waitKey(0) - cv2.destroyAllWindows() - - def draw_bounding_box_on_image(self,image,ymin,xmin,ymax,xmax,color,font,thickness=4,display_str_list=()): - """Adds a bounding box to an image.""" - draw = ImageDraw.Draw(image) - im_width, im_height = image.size - (left, right, top, bottom) = (xmin * im_width, xmax * im_width, - ymin * im_height, ymax * im_height) - draw.line([(left, top), (left, bottom), (right, bottom), (right, top), - (left, top)], - width=thickness, - fill=color) - # If the total height of the display strings added to the top of the bounding - # box exceeds the top of the image, stack the strings below the bounding box - # instead of above. - display_str_heights = [font.getsize(ds)[1] for ds in display_str_list] - # Each display_str has a top and bottom margin of 0.05x. - total_display_str_height = (1 + 2 * 0.05) * sum(display_str_heights) - - if top > total_display_str_height: - text_bottom = top - else: - text_bottom = top + total_display_str_height - # Reverse list and print from bottom to top. - for display_str in display_str_list[::-1]: - text_width, text_height = font.getsize(display_str) - margin = np.ceil(0.05 * text_height) - draw.rectangle([(left, text_bottom - text_height - 2 * margin), - (left + text_width, text_bottom)], - fill=color) - draw.text((left + margin, text_bottom - text_height - margin), - display_str, - fill="black", - font=font) - text_bottom -= text_height - 2 * margin - - # create list of dictionary containing predictions - def create_predictions_list(self, class_ids, confidences, boxes): - for i in range(len(class_ids)): - obj_dict = { - "class_id": class_ids[i], - "confidence": confidences[i], - "box": boxes[i] - } - self.predictions.append(obj_dict) - - def draw_boxes(self,image,boxes,class_ids,confidences,max_boxes=10): - """Overlay labeled boxes on an image with formatted scores and label names.""" - colors = list(ImageColor.colormap.values()) - - try: - font = ImageFont.truetype("/usr/share/fonts/truetype/liberation/LiberationSansNarrow-Regular.ttf", - 25) - except IOError: - print("Font not found, using default font.") - font = ImageFont.load_default() - - for i in range(min(boxes.shape[0], max_boxes)): - if confidences[i] >= self.conf: - ymin, xmin, ymax, xmax = tuple(boxes[i]) - display_str = "{}: {}%".format(self.labels[class_ids[i]], int(100 * confidences[i])) - color = colors[hash(class_ids[i]) % len(colors)] - image_pil = Image.fromarray(np.uint8(image)).convert("RGB") - self.draw_bounding_box_on_image(image_pil,ymin,xmin,ymax,xmax,color,font,display_str_list=[display_str]) - np.copyto(image, np.array(image_pil)) - return image - - def load_img(self,path): - img = tf.io.read_file(path) - img = tf.image.decode_jpeg(img, channels=3) - return img - - def get_predictions(self,cv_image): - - if cv_image is None: - # TODO: show warning message (different color, maybe) - return None,None - - else : - #Convert img to RGB - self.frame = cv_image - - self.frame_count += 1 - self.total_frames += 1 - - rgb = cv2.cvtColor(self.frame, cv2.COLOR_BGR2RGB) - - # COnverting to uint8 - rgb_tensor = tf.convert_to_tensor(rgb, dtype=tf.uint8) - - #Add dims to rgb_tensor - rgb_tensor = tf.expand_dims(rgb_tensor , 0) - - - result = self.detector(rgb_tensor) - result = {key:value.numpy() for key,value in result.items()} - - self.create_predictions_list(result["detection_boxes"][0],result["detection_classes"][0], result["detection_scores"][0]) - image_with_boxes = self.draw_boxes(cv_image,result["detection_boxes"][0],result["detection_classes"][0], result["detection_scores"][0]) - - # fps - if self.frame_count >= 30: - self.end = time.time_ns() - self.fps = 1000000000 * self.frame_count / (self.end - self.start) - self.frame_count = 0 - self.start = time.time_ns() - - if self.fps > 0: - self.fps_label = "FPS: %.2f" % self.fps - cv2.putText(self.frame, self.fps_label, (10, 25), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2) - - return [self.predictions, image_with_boxes] - - - def detect_img(self,image_url): - start_time = time.time() - self.run_detector(self.detector, image_url)#Convert img to RGB - rgb = cv2.cvtColor(cv_image, cv2.COLOR_BGR2RGB) - # COnverting to uint8 - rgb_tensor = tf.convert_to_tensor(rgb, dtype=tf.uint8) - #Add dims to rgb_tensor - rgb_tensor = tf.expand_dims(rgb_tensor , 0) - start_time = time.time() - result = self.detector(rgb_tensor) - end_time = time.time() - result = {key:value.numpy() for key,value in result.items()} - print("Found %d objects." % len(result["detection_scores"])) - print("Inference time: ", end_time-start_time) - self.create_predictions_list(cv_image,result["detection_boxes"][0],result["detection_classes"][0], result["detection_scores"][0]) - image_with_boxes = self.draw_boxes(cv_image,result["detection_boxes"][0],result["detection_classes"][0], result["detection_scores"][0]) - self.display_image(self.predictions,image_with_boxes) - - end_time = time.time() - print("Inference time:",end_time-start_time) - -if __name__=='__main__': - # Load model - det = EfficientDet() - det.detect_img("/home/sanchay/yolo_catkin/src/yolov8_test/scripts/dog_cat.jpg") \ No newline at end of file diff --git a/object_detection/object_detection/Detectors/RetinaNet.py b/object_detection/object_detection/Detectors/RetinaNet.py index 8a54122..1005034 100755 --- a/object_detection/object_detection/Detectors/RetinaNet.py +++ b/object_detection/object_detection/Detectors/RetinaNet.py @@ -1,122 +1,75 @@ -#!/usr/bin/env python3 +# Copyright (c) 2023 A.T.O.M ROBOTICS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. -from tensorflow import keras -from keras_retinanet import models -from keras_retinanet.utils.image import read_image_bgr, preprocess_image, resize_image -from keras_retinanet.utils.visualization import draw_box, draw_caption -from keras_retinanet.utils.colors import label_color -import matplotlib.pyplot as plt -import cv2 import os -import numpy as np -import time -import matplotlib.pyplot as plt - -class RetinaNet: - def __init__(self, model_dir_path, weight_file_name, conf_threshold = 0.7, - score_threshold = 0.4, nms_threshold = 0.25, is_cuda = 0, show_fps = 1): - - self.model_dir_path = model_dir_path - self.weight_file_name = weight_file_name +from keras_retinanet import models +from keras_retinanet.utils.image import preprocess_image, resize_image +import numpy as np - self.predictions = [] - self.conf_threshold = conf_threshold - self.show_fps = show_fps - self.is_cuda = is_cuda +from ..DetectorBase import DetectorBase - if self.show_fps : - self.frame_count = 0 - self.total_frames = 0 - self.fps = -1 - self.start = time.time_ns() - - self.labels_to_names = self.load_classes() - self.build_model() - def build_model(self) : +class RetinaNet(DetectorBase): - try : - self.model_path = os.path.join(self.model_dir_path, self.weight_file_name) - self.model = models.load_model(self.model_path, backbone_name='resnet50') + def __init(self): + super.__init__() - except : - raise Exception("Error loading given model from path: {}. Maybe the file doesn't exist?".format(self.model_path)) + def build_model(self, model_dir_path, weight_file_name): + model_path = os.path.join(model_dir_path, weight_file_name) + try: + self.model = models.load_model(model_path, backbone_name='resnet50') + except Exception as e: + print("Loading the model failed with exception {}".format(e)) + raise Exception("Error loading given model from path: {}.".format(model_path) + + "Maybe the file doesn't exist?") - def load_classes(self): + def load_classes(self, model_dir_path): self.class_list = [] - - with open(self.model_dir_path + "/classes.txt", "r") as f: - self.class_list = [cname.strip() for cname in f.readlines()] - - return self.class_list - - def create_predictions_list(self, class_ids, confidences, boxes): - for i in range(len(class_ids)): - obj_dict = { - "class_id": class_ids[i], - "confidence": confidences[i], - "box": boxes[i] - } - self.predictions.append(obj_dict) - + with open(model_dir_path + "/classes.txt", "r") as f: + self.class_list = [cname.strip() for cname in f.readlines()] - def get_predictions(self, cv_image): + return self.class_list + def get_predictions(self, cv_image): if cv_image is None: # TODO: show warning message (different color, maybe) - return None,None - - else : - + return None + else: # copy to draw on self.frame = cv_image.copy() # preprocess image for network - input = preprocess_image(self.frame) - input, scale = resize_image(input) - - self.frame_count += 1 - self.total_frames += 1 - + processed_img = preprocess_image(self.frame) + processed_img, scale = resize_image(processed_img) + # process image - start = time.time() - boxes, scores, labels = self.model.predict_on_batch(np.expand_dims(input, axis=0)) - #print("processing time: ", time.time() - start) - + boxes_all, confidences_all, class_ids_all = self.model.predict_on_batch(np.expand_dims(processed_img, axis=0)) + + boxes, confidences, class_ids = [], [], [] + + for index in range(len(confidences_all[0])): + if confidences_all[0][index] != -1: + confidences.append(confidences_all[0][index]) + boxes.append(boxes_all[0][index]) + class_ids.append(class_ids_all[0][index]) + # correct for image scale - boxes /= scale - - self.create_predictions_list(labels, scores, boxes) - - # visualize detections - for box, score, label in zip(boxes[0], scores[0], labels[0]): - # scores are sorted so we can break - if score < self.conf_threshold: - break - - color = label_color(label) - - b = box.astype(int) - draw_box(self.frame, b, color=color) - - caption = "{} {:.3f}".format(self.labels_to_names[label], score) - #print(self.labels_to_names[label]) - draw_caption(self.frame, b, caption) - - if self.show_fps : - if self.frame_count >= 30: - self.end = time.time_ns() - self.fps = 1000000000 * self.frame_count / (self.end - self.start) - self.frame_count = 0 - self.start = time.time_ns() - - if self.fps > 0: - self.fps_label = "FPS: %.2f" % self.fps - cv2.putText(self.frame, self.fps_label, (10, 25), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2) - - return (self.predictions, self.frame) - - - \ No newline at end of file + # boxes = [x/scale for x in boxes] + boxes = [[int(coord/scale) for coord in box] for box in boxes] + + super().create_predictions_list(class_ids, confidences, boxes) + + return self.predictions diff --git a/object_detection/object_detection/Detectors/YOLOv5.py b/object_detection/object_detection/Detectors/YOLOv5.py old mode 100644 new mode 100755 index 571f3d0..4553004 --- a/object_detection/object_detection/Detectors/YOLOv5.py +++ b/object_detection/object_detection/Detectors/YOLOv5.py @@ -1,151 +1,65 @@ -import time +# Copyright (c) 2023 A.T.O.M ROBOTICS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import os -import cv2 -import numpy as np + +import torch from ..DetectorBase import DetectorBase class YOLOv5(DetectorBase): - def __init__(self, conf_threshold = 0.7, score_threshold = 0.4, nms_threshold = 0.25, is_cuda = 0): + def __init__(self, conf_threshold=0.7): super().__init__() + self.conf_threshold = conf_threshold - # opencv img input - self.frame = None - self.net = None - self.INPUT_WIDTH = 640 - self.INPUT_HEIGHT = 640 - self.CONFIDENCE_THRESHOLD = conf_threshold - - self.is_cuda = is_cuda - - - # load model and prepare its backend to either run on GPU or CPU, see if it can be added in constructor def build_model(self, model_dir_path, weight_file_name): - model_path = os.path.join(model_dir_path, weight_file_name) - try: - self.net = cv2.dnn.readNet(model_path) - except: - raise Exception("Error loading given model from path: {}. Maybe the file doesn't exist?".format(model_path)) + model_path = os.path.join(model_dir_path, weight_file_name) + self.model = torch.hub.load('ultralytics/yolov5:v6.0', 'custom', path=model_path, + force_reload=True) + except Exception as e: + print("Loading model failed with exception: {}".format(e)) + raise Exception("Error loading given model from path: {}.".format(model_path) + + " Maybe the file doesn't exist?") - if self.is_cuda: - print("is_cuda was set to True. Attempting to use CUDA") - self.net.setPreferableBackend(cv2.dnn.DNN_BACKEND_CUDA) - self.net.setPreferableTarget(cv2.dnn.DNN_TARGET_CUDA_FP16) - else: - print("is_cuda was set to False. Running on CPU") - self.net.setPreferableBackend(cv2.dnn.DNN_BACKEND_OPENCV) - self.net.setPreferableTarget(cv2.dnn.DNN_TARGET_CPU) - - - # load classes.txt that contains mapping of model with labels - # TODO: add try/except to raise exception that tells the use to check the name if it is classes.txt def load_classes(self, model_dir_path): self.class_list = [] - with open(model_dir_path + "/classes.txt", "r") as f: - self.class_list = [cname.strip() for cname in f.readlines()] - return self.class_list - - - def detect(self, image): - # convert image to 640x640 - blob = cv2.dnn.blobFromImage(image, 1/255.0, (self.INPUT_WIDTH, self.INPUT_HEIGHT), swapRB=True, crop=False) - self.net.setInput(blob) - preds = self.net.forward() - return preds - - - # extract bounding box, class IDs and confidences of detected objects - # YOLOv5 returns a 3D tensor of dimension 25200*(5 + n_classes) - def wrap_detection(self, input_image, output_data): - class_ids = [] - confidences = [] - boxes = [] - - rows = output_data.shape[0] - - image_width, image_height, _ = input_image.shape - - x_factor = image_width / self.INPUT_WIDTH - y_factor = image_height / self.INPUT_HEIGHT - - # Iterate through all the 25200 vectors - for r in range(rows): - row = output_data[r] - # Continue only if Pc > conf_threshold - confidence = row[4] - if confidence >= self.CONFIDENCE_THRESHOLD: - - # One-hot encoded vector representing class of object - classes_scores = row[5:] - - # Returns min and max values in a array alongwith their indices - _, _, _, max_indx = cv2.minMaxLoc(classes_scores) - - # Extract the column index of the maximum values in classes_scores - class_id = max_indx[1] - - # Continue of the class score is greater than a threshold - # class_score represents the probability of an object belonging to that class - if (classes_scores[class_id] > .25): - - confidences.append(confidence) - - class_ids.append(class_id) - - x, y, w, h = row[0].item(), row[1].item(), row[2].item(), row[3].item() - left = int((x - 0.5 * w) * x_factor) - top = int((y - 0.5 * h) * y_factor) - width = int(w * x_factor) - height = int(h * y_factor) - box = np.array([left, top, width, height]) - boxes.append(box) - - # removing intersecting bounding boxes - indexes = cv2.dnn.NMSBoxes(boxes, confidences, 0.25, 0.45) - - result_class_ids = [] - result_confidences = [] - result_boxes = [] - - for i in indexes: - result_confidences.append(confidences[i]) - result_class_ids.append(class_ids[i]) - result_boxes.append(boxes[i]) - - return result_class_ids, result_confidences, result_boxes - - - # makes image square with dimension max(h, w) - def format_yolov5(self): - row, col, _ = self.frame.shape - _max = max(col, row) - result = np.zeros((_max, _max, 3), np.uint8) - result[0:row, 0:col] = self.frame - return result + with open(os.path.join(model_dir_path, 'classes.txt')) as f: + self.class_list = [cname.strip() for cname in f.readlines()] + return self.class_list def get_predictions(self, cv_image): - #Clear list - self.predictions = [] - if cv_image is None: # TODO: show warning message (different color, maybe) - return None,None + return None, None else: self.frame = cv_image + class_id = [] + confidence = [] + boxes = [] + + results = self.model(self.frame) - # make image square - inputImage = self.format_yolov5() + for *xyxy, conf, label in results.xyxy[0]: + class_id.append(int(label)) + confidence.append(conf.item()) + boxes.append([int(xy) for xy in xyxy]) - outs = self.detect(inputImage) - class_ids, confidences, boxes = self.wrap_detection(inputImage, outs[0]) + super().create_predictions_list(class_id, confidence, boxes) - super().create_predictions_list(class_ids, confidences, boxes) - - print("Detected ids: ", class_ids) - - return self.predictions \ No newline at end of file + return self.predictions diff --git a/object_detection/object_detection/Detectors/YOLOv8.py b/object_detection/object_detection/Detectors/YOLOv8.py index f76c201..9bb1918 100755 --- a/object_detection/object_detection/Detectors/YOLOv8.py +++ b/object_detection/object_detection/Detectors/YOLOv8.py @@ -1,103 +1,66 @@ -import cv2 -from ultralytics import YOLO -import os -import time - -class YOLOv8: - def __init__(self, model_dir_path, weight_file_name, conf_threshold = 0.7, - score_threshold = 0.4, nms_threshold = 0.25, - show_fps = 1, is_cuda = 0): - - self.model_dir_path = model_dir_path - self.weight_file_name = weight_file_name - - - self.conf_threshold = conf_threshold - self.show_fps = show_fps - self.is_cuda = is_cuda - - #FPS - if self.show_fps : - self.frame_count = 0 - self.total_frames = 0 - self.fps = -1 - self.start = time.time_ns() - self.frame = None - +# Copyright (c) 2023 A.T.O.M ROBOTICS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. - self.predictions = [] - self.build_model() - self.load_classes() - - - def build_model(self) : - - try : - model_path = os.path.join(self.model_dir_path, self.weight_file_name) - self.model = YOLO(model_path) - - except : - raise Exception("Error loading given model from path: {}. Maybe the file doesn't exist?".format(model_path)) - - def load_classes(self): +import os - self.class_list = [] +from ultralytics import YOLO - with open(self.model_dir_path + "/classes.txt", "r") as f: - self.class_list = [cname.strip() for cname in f.readlines()] +from ..DetectorBase import DetectorBase - return self.class_list - # create list of dictionary containing predictions - def create_predictions_list(self, class_ids, confidences, boxes): - - for i in range(len(class_ids)): - obj_dict = { - "class_id": class_ids[i], - "confidence": confidences[i], - "box": boxes[i] - } - self.predictions.append(obj_dict) +class YOLOv8(DetectorBase): - def get_predictions(self, cv_image): + def __init__(self, conf_threshold=0.7): + super().__init__() + self.conf_threshold = conf_threshold - if cv_image is None: - # TODO: show warning message (different color, maybe) - return None,None - - else : - self.frame = cv_image - self.frame_count += 1 - self.total_frames += 1 + def build_model(self, model_dir_path, weight_file_name): + try: + model_path = os.path.join(model_dir_path, weight_file_name) + self.model = YOLO(model_path) + except Exception as e: + print("Loading model failed with exception: {}".format(e)) + raise Exception("Error loading given model from path: {}.".format(model_path) + + " Maybe the file doesn't exist?") - class_id = [] - confidence = [] - bb = [] - result = self.model.predict(self.frame, conf = self.conf_threshold) # Perform object detection on image - row = result[0].boxes + def load_classes(self, model_dir_path): + self.class_list = [] - for box in row: - class_id.append(box.cls) - confidence.append(box.conf) - bb.append(box.xyxy) + with open(model_dir_path + "/classes.txt", "r") as f: + self.class_list = [cname.strip() for cname in f.readlines()] - self.create_predictions_list(class_id,confidence,bb) - result = self.model.predict(self.frame, conf = self.conf_threshold) - output_frame = result[0].plot() # Frame with bounding boxes + return self.class_list - print("frame_count : ", self.frame_count) + def get_predictions(self, cv_image): + if cv_image is None: + # TODO: show warning message (different color, maybe) + return None, None + else: + self.frame = cv_image + class_id = [] + confidence = [] + boxes = [] + # Perform object detection on image + result = self.model.predict(self.frame, conf=self.conf_threshold, verbose=False) + row = result[0].boxes.cpu() - if self.show_fps : - if self.frame_count >= 30: - self.end = time.time_ns() - self.fps = 1000000000 * self.frame_count / (self.end - self.start) - self.frame_count = 0 - self.start = time.time_ns() + for box in row: + class_id.append(box.cls.numpy().tolist()[0]) + confidence.append(box.conf.numpy().tolist()[0]) + boxes.append(box.xyxy.numpy().tolist()[0]) - if self.fps > 0: - self.fps_label = "FPS: %.2f" % self.fps - cv2.putText(output_frame, self.fps_label, (10, 25), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2) + super().create_predictions_list(class_id, confidence, boxes) - return self.predictions, output_frame - \ No newline at end of file + return self.predictions diff --git a/object_detection/object_detection/ObjectDetection.py b/object_detection/object_detection/ObjectDetection.py index 3bbf7ca..844e09f 100644 --- a/object_detection/object_detection/ObjectDetection.py +++ b/object_detection/object_detection/ObjectDetection.py @@ -1,38 +1,53 @@ #! /usr/bin/env python3 -import os +# Copyright (c) 2023 A.T.O.M ROBOTICS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import importlib +import os +import time + +import cv2 + +from cv_bridge import CvBridge import rclpy from rclpy.node import Node from sensor_msgs.msg import Image -#from vision_msgs.msg import BoundingBox2D - -from cv_bridge import CvBridge -import cv2 class ObjectDetection(Node): + def __init__(self): super().__init__('object_detection') # create an empty list that will hold the names of all available detector self.available_detectors = [] - + # fill available_detectors with the detectors from Detectors dir self.discover_detectors() self.declare_parameters( namespace='', parameters=[ - - ('input_img_topic', ""), - ('output_bb_topic', ""), - ('output_img_topic', ""), - ('model_params.detector_type', ""), - ('model_params.model_dir_path', ""), - ('model_params.weight_file_name', ""), + ('input_img_topic', ''), + ('output_bb_topic', ''), + ('output_img_topic', ''), + ('model_params.detector_type', ''), + ('model_params.model_dir_path', ''), + ('model_params.weight_file_name', ''), ('model_params.confidence_threshold', 0.7), ('model_params.show_fps', 1), ] @@ -42,84 +57,110 @@ def __init__(self): self.input_img_topic = self.get_parameter('input_img_topic').value self.output_bb_topic = self.get_parameter('output_bb_topic').value self.output_img_topic = self.get_parameter('output_img_topic').value - + # model params self.detector_type = self.get_parameter('model_params.detector_type').value self.model_dir_path = self.get_parameter('model_params.model_dir_path').value self.weight_file_name = self.get_parameter('model_params.weight_file_name').value self.confidence_threshold = self.get_parameter('model_params.confidence_threshold').value self.show_fps = self.get_parameter('model_params.show_fps').value - - # raise an exception if specified detector was not found - if self.detector_type not in self.available_detectors: - raise ModuleNotFoundError(self.detector_type + " Detector specified in config was not found. " + - "Check the Detectors dir for available detectors.") - else: - self.load_detector() - - + + # Load the detector + self.load_detector() + self.img_pub = self.create_publisher(Image, self.output_img_topic, 10) self.bb_pub = None self.img_sub = self.create_subscription(Image, self.input_img_topic, self.detection_cb, 10) self.bridge = CvBridge() - + if self.show_fps: + self.start_time = time.time() + self.frame_count = 0 + self.total_elapsed_time = 0 + self.average_fps = 0 + def discover_detectors(self): curr_dir = os.path.dirname(__file__) - dir_contents = os.listdir(curr_dir + "/Detectors") + dir_contents = os.listdir(curr_dir + "/Detectors") for entity in dir_contents: if entity.endswith('.py'): self.available_detectors.append(entity[:-3]) self.available_detectors.remove('__init__') - - + def load_detector(self): - detector_mod = importlib.import_module(".Detectors." + self.detector_type, "object_detection") - detector_class = getattr(detector_mod, self.detector_type) - self.detector = detector_class() - - self.detector.build_model(self.model_dir_path, self.weight_file_name) - self.detector.load_classes(self.model_dir_path) - - print("Your detector : {} has been loaded !".format(self.detector_type)) - - + for detector_name in self.available_detectors: + if self.detector_type.lower() == detector_name.lower(): + + detector_mod = importlib.import_module(".Detectors." + detector_name, + "object_detection") + detector_class = getattr(detector_mod, detector_name) + self.detector = detector_class() + + self.detector.build_model(self.model_dir_path, self.weight_file_name) + self.detector.load_classes(self.model_dir_path) + + print("Your detector: {} has been loaded !".format(detector_name)) + return + + raise ModuleNotFoundError(self.detector_type + " Detector specified in config was not found. " + + "Check the Detectors dir for available detectors.") + def detection_cb(self, img_msg): cv_image = self.bridge.imgmsg_to_cv2(img_msg, "bgr8") + start_time = time.time() + predictions = self.detector.get_predictions(cv_image=cv_image) - if predictions == None : - print("Image input from topic : {} is empty".format(self.input_img_topic)) - else : + if predictions is None: + print("Image input from topic: {} is empty".format(self.input_img_topic)) + else: for prediction in predictions: - left, top, width, height = prediction['box'] - right = left + width - bottom = top + height + x1, y1, x2, y2 = map(int, prediction['box']) + + # Draw the bounding box + cv_image = cv2.rectangle(cv_image, (x1, y1), (x2, y2), (0, 255, 0), 1) - #Draw the bounding box - cv_image = cv2.rectangle(cv_image,(left,top),(right, bottom),(0,255,0),1) + # Show names of classes on the output image + class_id = int(prediction['class_id']) + class_name = self.detector.class_list[class_id] + label = f"{class_name}: {prediction['confidence']:.2f}" + + cv_image = cv2.putText(cv_image, label, (x1, y1 - 5), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 1) + + if self.show_fps: + elapsed_time = time.time() - start_time + self.frame_count += 1 + self.total_elapsed_time += elapsed_time + + # Write FPS on the frame + cv_image = cv2.putText(cv_image, f"FPS: {self.average_fps:.2f}", (10, 30), + cv2.FONT_HERSHEY_SIMPLEX, 0.8, (0, 255, 0), 2) + + if time.time() - self.start_time >= 1.0: + self.average_fps = self.frame_count / self.total_elapsed_time + + self.frame_count = 0 + self.total_elapsed_time = 0 + self.start_time = time.time() output = self.bridge.cv2_to_imgmsg(cv_image, "bgr8") self.img_pub.publish(output) - print(predictions) def main(): rclpy.init() od = ObjectDetection() - try : + try: rclpy.spin(od) except Exception as e: print(e) -if __name__=="__main__" : +if __name__ == "__main__": main() - - - diff --git a/object_detection/package.xml b/object_detection/package.xml index 068e66a..1bd237b 100644 --- a/object_detection/package.xml +++ b/object_detection/package.xml @@ -3,15 +3,17 @@ object_detection 0.0.0 - TODO: Package description + This is a ROS 2 package aimed at providing an interfacing between Deep Learning models and the ROS architecture using a plug-and-play modular architecture singh - TODO: License declaration + Apache 2.0 ament_copyright ament_flake8 ament_pep257 python3-pytest - + vision_msgs + cv_bridge + ament_python diff --git a/object_detection/requirements.txt b/object_detection/requirements.txt index 71ac88e..06197c0 100644 --- a/object_detection/requirements.txt +++ b/object_detection/requirements.txt @@ -1,9 +1,9 @@ +tensorflow==2.14.0 +tensorflow-hub==0.13.0 keras-retinanet==1.0.0 -matplotlib==3.5.4 -numpy==1.25.0 +matplotlib==3.7.2 +numpy==1.26.1 opencv-python==4.7.0.72 pandas==2.0.3 pillow==9.5.0 -tensorflow==2.12.0 -tensorflow-hub==0.13.0 -ultralytics==8.0.124 \ No newline at end of file +ultralytics==8.0.124 diff --git a/object_detection/setup.cfg b/object_detection/setup.cfg index 90b06b2..2165e16 100644 --- a/object_detection/setup.cfg +++ b/object_detection/setup.cfg @@ -2,3 +2,9 @@ script_dir=$base/lib/object_detection [install] install_scripts=$base/lib/object_detection +[flake8] +extend-ignore = B902,C816,D100,D101,D102,D103,D104,D105,D106,D107,D203,D212,D404,I202,Q000 +import-order-style = google +max-line-length = 125 +show-source = true +statistics = true \ No newline at end of file diff --git a/object_detection/setup.py b/object_detection/setup.py index 6cec5fe..5602255 100644 --- a/object_detection/setup.py +++ b/object_detection/setup.py @@ -1,6 +1,7 @@ -from setuptools import setup -import os from glob import glob +import os + +from setuptools import setup package_name = 'object_detection' @@ -20,8 +21,8 @@ zip_safe=True, maintainer='singh', maintainer_email='jasmeet0915@gmail.com', - description='TODO: Package description', - license='TODO: License declaration', + description='Plug-and-Play ROS 2 package for Perception in Robotics', + license='Apache 2.0', tests_require=['pytest'], entry_points={ 'console_scripts': [ diff --git a/object_detection/test/test_flake8.py b/object_detection/test/test_flake8.py index 27ee107..7932fbd 100644 --- a/object_detection/test/test_flake8.py +++ b/object_detection/test/test_flake8.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from os.path import abspath, dirname, join + from ament_flake8.main import main_with_errors import pytest @@ -19,7 +21,12 @@ @pytest.mark.flake8 @pytest.mark.linter def test_flake8(): - rc, errors = main_with_errors(argv=[]) + config_file = join(dirname(dirname(dirname(abspath(__file__)))), ".flake8") + args = [ + "--config={}".format(config_file) + ] + + rc, errors = main_with_errors(argv=args) assert rc == 0, \ 'Found %d code style errors / warnings:\n' % len(errors) + \ '\n'.join(errors) diff --git a/perception_bringup/launch/playground.launch.py b/perception_bringup/launch/playground.launch.py index 83868b0..a2bd10c 100644 --- a/perception_bringup/launch/playground.launch.py +++ b/perception_bringup/launch/playground.launch.py @@ -1,10 +1,10 @@ -# Copyright (c) 2018 Intel Corporation +# Copyright (c) 2023 A.T.O.M ROBOTICS # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -18,56 +18,48 @@ from ament_index_python.packages import get_package_share_directory from launch import LaunchDescription -from launch.actions import IncludeLaunchDescription, DeclareLaunchArgument -from launch.substitutions import LaunchConfiguration +from launch.actions import DeclareLaunchArgument, IncludeLaunchDescription from launch.launch_description_sources import PythonLaunchDescriptionSource from launch_ros.actions import Node def generate_launch_description(): - - pkg_perception_bringup = get_package_share_directory("perception_bringup") - #pkg_ros_gz_sim = get_package_share_directory("ros_gz_sim") - - world_name = "playground" - - for arg in sys.argv: - if arg.startswith("world:="): - world_name = arg.split(":=")[1] - - world_sdf = pkg_perception_bringup + "/worlds/" + world_name + ".sdf" - - '''gz_sim = IncludeLaunchDescription( - PythonLaunchDescriptionSource( - os.path.join(pkg_ros_gz_sim, 'launch', 'gz_sim.launch.py')), - )''' - - gz_sim_share = get_package_share_directory("ros_gz_sim") - gz_sim = IncludeLaunchDescription( - PythonLaunchDescriptionSource(os.path.join(gz_sim_share, "launch", "gz_sim.launch.py")), - launch_arguments={ - "gz_args" : world_sdf - }.items() - ) - - parameter_bridge = Node(package="ros_gz_bridge", executable="parameter_bridge", - parameters = [ - {'config_file' : os.path.join(pkg_perception_bringup, "config", "bridge.yaml")} - ] - ) - - arg_gz_sim = DeclareLaunchArgument('gz_args', default_value=world_sdf) - - arg_world_name = DeclareLaunchArgument('world', default_value='playground_world' ) - - launch = [ - gz_sim, - parameter_bridge - ] - - args = [ - arg_gz_sim, - arg_world_name - ] - - return LaunchDescription(args + launch) + pkg_perception_bringup = get_package_share_directory("perception_bringup") + pkg_ros_gz_sim = get_package_share_directory("ros_gz_sim") + ros_gz_bridge_config = os.path.join(pkg_perception_bringup, "config", "bridge.yaml") + world_name = "playground" + + for arg in sys.argv: + if arg.startswith("world:="): + world_name = arg.split(":=")[1] + + world_sdf = pkg_perception_bringup + "/worlds/" + world_name + ".sdf" + + gz_sim = IncludeLaunchDescription( + PythonLaunchDescriptionSource( + os.path.join(pkg_ros_gz_sim, "launch", "gz_sim.launch.py") + ), + launch_arguments={ + "gz_args": world_sdf + }.items() + ) + + parameter_bridge = Node(package="ros_gz_bridge", executable="parameter_bridge", + parameters=[ + {'config_file': ros_gz_bridge_config} + ]) + + arg_gz_sim = DeclareLaunchArgument('gz_args', default_value=world_sdf) + arg_world_name = DeclareLaunchArgument('world', default_value='playground_world') + + launch = [ + gz_sim, + parameter_bridge + ] + + args = [ + arg_gz_sim, + arg_world_name + ] + + return LaunchDescription(args + launch) diff --git a/perception_bringup/package.xml b/perception_bringup/package.xml index fafefbc..a4894b2 100644 --- a/perception_bringup/package.xml +++ b/perception_bringup/package.xml @@ -5,13 +5,14 @@ 0.0.0 TODO: Package description singh - TODO: License declaration + Apache 2.0 ament_cmake ament_lint_auto ament_lint_common - + ros_gz + ros_gz_bridge ament_cmake