Merge branch 'master' of https://github.com/twinsant/Auto-GPT

.dockerignore

@@ -37,3 +37,4 @@ rnd/autogpt_builder/.env.local
 rnd/autogpt_server/.env
 rnd/autogpt_server/.venv/
 
+rnd/market/.env

.github/workflows/autogpt-server-ci.yml

@@ -128,9 +128,14 @@ jobs:
 
       - name: Run pytest with coverage
         run: |
-          poetry run pytest -vv \
-            test
+          if [[ "${{ runner.debug }}" == "1" ]]; then
+            poetry run pytest -vv -o log_cli=true -o log_cli_level=DEBUG test
+          else
+            poetry run pytest -vv test
+          fi
         if: success() || (failure() && steps.lint.outcome == 'failure')
+        env:
+          LOG_LEVEL: ${{ runner.debug && 'DEBUG' || 'INFO' }}
     env:
       CI: true
       PLAIN_OUTPUT: True

docs/content/forge/components/built-in-components.md

@@ -69,6 +69,8 @@ Lets the agent execute non-interactive Shell commands and Python code. Python ex
 | `shell_denylist`         | List of prohibited shell commands                    | `List[str]`                 | `[]`              |
 | `docker_container_name`  | Name of the Docker container used for code execution | `str`                       | `"agent_sandbox"` |
 
+All shell command configurations are expected to be for convience only. This component is not secure and should not be used in production environments. It is recommended to use more appropriate sandboxing.
+
 ### CommandProvider
 
 - `execute_shell` execute shell command

docs/content/server/setup.md

@@ -73,6 +73,7 @@ Once you have installed Yarn and Poetry, you can run the following command to in
 
 ```bash
 cd rnd/autogpt_server
+cp .env.example .env
 poetry install
 ```
 
@@ -90,7 +91,7 @@ Once you have installed the dependencies, you can proceed to the next step.
 In order to setup the database, you need to run the following commands, in the same terminal you ran the `poetry install` command:
 
    ```sh
-   docker compose up postgres -d
+   docker compose up postgres redis -d
    poetry run prisma migrate dev
    ```
 After deploying the migration, to ensure that the database schema is correctly mapped to your codebase, allowing the application to interact with the database properly, you need to generate the Prisma database model:
@@ -101,7 +102,15 @@ poetry run prisma generate
 
 Without running this command, the necessary Python modules (prisma.models) won't be available, leading to a `ModuleNotFoundError`.
 
-### Running the server
+### Running the server without Docker
+
+To run the server, you can run the following commands in the same terminal you ran the `poetry install` command:
+
+```bash
+poetry run app
+```
+
+### Running the server within Docker
 
 To run the server, you can run the following commands in the same terminal you ran the `poetry install` command:
 
@@ -110,7 +119,7 @@ docker compose build
 docker compose up
 ```
 
-In the other terminal, you can run the following command to start the frontend:
+In the other terminal from autogpt_builder, you can run the following command to start the frontend:
 
 ```bash
 yarn dev
@@ -119,3 +128,10 @@ yarn dev
 ### Checking if the server is running
 
 You can check if the server is running by visiting [http://localhost:3000](http://localhost:3000) in your browser.
+
+### Notes: 
+By default the daemons for different services run on the following ports: 
+
+Execution Manager Daemon: 8002
+Execution Scheduler Daemon: 8003
+Rest Server Daemon: 8004

rnd/README.md

@@ -1,36 +1,114 @@
-This is a guide to setting up and running the AutoGPT Server and Builder. This tutorial will cover downloading the necessary files, setting up the server, and testing the system.
-
-https://github.com/user-attachments/assets/fd0d0f35-3155-4263-b575-ba3efb126cb4
-
-1. Navigate to the AutoGPT GitHub repository.
-2. Click the "Code" button, then select "Download ZIP".
-3. Once downloaded, extract the ZIP file to a folder of your choice.
-
-4. Open the extracted folder and navigate to the "rnd" directory.
-5. Enter the "AutoGPT server" folder.
-6. Open a terminal window in this directory.
-7. Locate and open the README file in the AutoGPT server folder: [doc](./autogpt_server/README.md#setup).
-8. Copy and paste each command from the setup section in the README into your terminal.
-   - Important: Wait for each command to finish before running the next one.
-9. If all commands run without errors, enter the final command: `poetry run app`
-10. You should now see the server running in your terminal.
-
-11. Navigate back to the "rnd" folder.
-12. Open the "AutoGPT builder" folder.
-13. Open the README file in this folder: [doc](./autogpt_builder/README.md#getting-started).
-14. In your terminal, run the following commands:
-    ```
-    npm install
-    ```
-    ```
-    npm run dev
-    ```
-
-15. Once the front-end is running, click the link to navigate to `localhost:3000`.
-16. Click on the "Build" option.
-17. Add a few blocks to test the functionality.
-18. Connect the blocks together.
-19. Click "Run".
-20. Check your terminal window - you should see that the server has received the request, is processing it, and has executed it.
-
-And there you have it! You've successfully set up and tested AutoGPT.
+# AutoGPT Platform
+
+Welcome to the AutoGPT Platform - a powerful system for creating and running AI agents to solve business problems. This platform enables you to harness the power of artificial intelligence to automate tasks, analyze data, and generate insights for your organization.
+
+## Getting Started
+
+### Prerequisites
+
+- Docker
+- Docker Compose V2 (comes with Docker Desktop, or can be installed separately)
+
+### Running the System
+
+To run the AutoGPT Platform, follow these steps:
+
+1. Clone this repository to your local machine.
+2. Navigate to the project directory.
+3. Run the following command:
+
+   ```
+   docker compose up -d
+   ```
+
+   This command will start all the necessary services defined in the `docker-compose.yml` file in detached mode.
+
+### Docker Compose Commands
+
+Here are some useful Docker Compose commands for managing your AutoGPT Platform:
+
+- `docker compose up -d`: Start the services in detached mode.
+- `docker compose stop`: Stop the running services without removing them.
+- `docker compose rm`: Remove stopped service containers.
+- `docker compose build`: Build or rebuild services.
+- `docker compose down`: Stop and remove containers, networks, and volumes.
+- `docker compose watch`: Watch for changes in your services and automatically update them.
+
+
+### Sample Scenarios
+
+Here are some common scenarios where you might use multiple Docker Compose commands:
+
+1. Updating and restarting a specific service:
+   ```
+   docker compose build api_srv
+   docker compose up -d --no-deps api_srv
+   ```
+   This rebuilds the `api_srv` service and restarts it without affecting other services.
+
+2. Viewing logs for troubleshooting:
+   ```
+   docker compose logs -f api_srv ws_srv
+   ```
+   This shows and follows the logs for both `api_srv` and `ws_srv` services.
+
+3. Scaling a service for increased load:
+   ```
+   docker compose up -d --scale executor=3
+   ```
+   This scales the `executor` service to 3 instances to handle increased load.
+
+4. Stopping the entire system for maintenance:
+   ```
+   docker compose stop
+   docker compose rm -f
+   docker compose pull
+   docker compose up -d
+   ```
+   This stops all services, removes containers, pulls the latest images, and restarts the system.
+
+5. Developing with live updates:
+   ```
+   docker compose watch
+   ```
+   This watches for changes in your code and automatically updates the relevant services.
+
+6. Checking the status of services:
+   ```
+   docker compose ps
+   ```
+   This shows the current status of all services defined in your docker-compose.yml file.
+
+These scenarios demonstrate how to use Docker Compose commands in combination to manage your AutoGPT Platform effectively.
+
+
+### Persisting Data
+
+To persist data for PostgreSQL and Redis, you can modify the `docker-compose.yml` file to add volumes. Here's how:
+
+1. Open the `docker-compose.yml` file in a text editor.
+2. Add volume configurations for PostgreSQL and Redis services:
+
+   ```yaml
+   services:
+     postgres:
+       # ... other configurations ...
+       volumes:
+         - postgres_data:/var/lib/postgresql/data
+
+     redis:
+       # ... other configurations ...
+       volumes:
+         - redis_data:/data
+
+   volumes:
+     postgres_data:
+     redis_data:
+   ```
+
+3. Save the file and run `docker compose up -d` to apply the changes.
+
+This configuration will create named volumes for PostgreSQL and Redis, ensuring that your data persists across container restarts.
+
+
+

rnd/autogpt_builder/.env.example

@@ -1,6 +1,6 @@
 NEXT_PUBLIC_AGPT_SERVER_URL=http://localhost:8000/api
 NEXT_PUBLIC_AGPT_WS_SERVER_URL=ws://localhost:8001/ws
-NEXT_PUBLIC_AGPT_MARKETPLACE_URL=http://localhost:8001/api/v1/market
+NEXT_PUBLIC_AGPT_MARKETPLACE_URL=http://localhost:8005/api/v1/market
 
 ## Supabase credentials
 ## YOU ONLY NEED THEM IF YOU WANT TO USE SUPABASE USER AUTHENTICATION

rnd/autogpt_builder/Dockerfile

@@ -1,19 +1,19 @@
 # Base stage for both dev and prod
 FROM node:21-alpine AS base
 WORKDIR /app
-COPY autogpt_builder/package.json autogpt_builder/yarn.lock ./
+COPY rnd/autogpt_builder/package.json rnd/autogpt_builder/yarn.lock ./
 RUN yarn install --frozen-lockfile
 
 # Dev stage
 FROM base AS dev
 ENV NODE_ENV=development
-COPY autogpt_builder/ .
+COPY rnd/autogpt_builder/ .
 EXPOSE 3000
-CMD ["npm", "run", "dev"]
+CMD ["yarn", "run", "dev"]
 
 # Build stage for prod
 FROM base AS build
-COPY autogpt_builder/ .
+COPY rnd/autogpt_builder/ .
 RUN npm run build
 
 # Prod stage

rnd/autogpt_builder/package.json

@@ -29,6 +29,7 @@
     "@radix-ui/react-tooltip": "^1.1.2",
     "@supabase/ssr": "^0.4.0",
     "@supabase/supabase-js": "^2.45.0",
+    "@tanstack/react-table": "^8.20.5",
     "@xyflow/react": "^12.1.0",
     "ajv": "^8.17.1",
     "class-variance-authority": "^0.7.0",
@@ -47,8 +48,8 @@
     "react-icons": "^5.2.1",
     "react-markdown": "^9.0.1",
     "react-modal": "^3.16.1",
-    "recharts": "^2.12.7",
     "react-shepherd": "^6.1.1",
+    "recharts": "^2.12.7",
     "tailwind-merge": "^2.3.0",
     "tailwindcss-animate": "^1.0.7",
     "uuid": "^10.0.0",

rnd/autogpt_builder/src/app/admin/marketplace/page.tsx

@@ -2,15 +2,18 @@ import { withRoleAccess } from "@/lib/withRoleAccess";
 
 import React from "react";
 import { getReviewableAgents } from "@/components/admin/marketplace/actions";
-import AdminMarketplaceCard from "@/components/admin/marketplace/AdminMarketplaceCard";
 import AdminMarketplaceAgentList from "@/components/admin/marketplace/AdminMarketplaceAgentList";
+import AdminFeaturedAgentsControl from "@/components/admin/marketplace/AdminFeaturedAgentsControl";
+import { Separator } from "@/components/ui/separator";
 async function AdminMarketplace() {
-  const agents = await getReviewableAgents();
+  const reviewableAgents = await getReviewableAgents();
+
   return (
-    <div>
-      <h3>Agents to review</h3>
-      <AdminMarketplaceAgentList agents={agents.agents} />
-    </div>
+    <>
+      <AdminMarketplaceAgentList agents={reviewableAgents.agents} />
+      <Separator className="my-4" />
+      <AdminFeaturedAgentsControl className="mt-4" />
+    </>
   );
 }
 

rnd/autogpt_builder/src/app/error.tsx

@@ -0,0 +1,43 @@
+"use client";
+
+import { useEffect } from "react";
+import { IconCircleAlert } from "@/components/ui/icons";
+import { Button } from "@/components/ui/button";
+import Link from "next/link";
+
+export default function Error({
+  error,
+  reset,
+}: {
+  error: Error & { digest?: string };
+  reset: () => void;
+}) {
+  useEffect(() => {
+    console.error(error);
+  }, [error]);
+
+  return (
+    <div className="fixed inset-0 flex items-center justify-center bg-background">
+      <div className="w-full max-w-md px-4 text-center sm:px-6">
+        <div className="mx-auto flex size-12 items-center justify-center rounded-full bg-muted">
+          <IconCircleAlert className="size-10" />
+        </div>
+        <h1 className="mt-8 text-2xl font-bold tracking-tight text-foreground">
+          Oops, something went wrong!
+        </h1>
+        <p className="mt-4 text-muted-foreground">
+          We&apos;re sorry, but an unexpected error has occurred. Please try
+          again later or contact support if the issue persists.
+        </p>
+        <div className="mt-6 flex flex-row justify-center gap-4">
+          <Button onClick={reset} variant="outline">
+            Retry
+          </Button>
+          <Button>
+            <Link href="/">Go to Homepage</Link>
+          </Button>
+        </div>
+      </div>
+    </div>
+  );
+}

rnd/autogpt_builder/src/app/loading.tsx

@@ -0,0 +1,21 @@
+import AgentFlowListSkeleton from "@/components/monitor/skeletons/AgentFlowListSkeleton";
+import React from "react";
+import FlowRunsListSkeleton from "@/components/monitor/skeletons/FlowRunsListSkeleton";
+import FlowRunsStatusSkeleton from "@/components/monitor/skeletons/FlowRunsStatusSkeleton";
+
+export default function MonitorLoadingSkeleton() {
+  return (
+    <div className="space-y-4 p-4">
+      <div className="grid grid-cols-1 gap-4 md:grid-cols-3">
+        {/* Agents Section */}
+        <AgentFlowListSkeleton />
+
+        {/* Runs Section */}
+        <FlowRunsListSkeleton />
+
+        {/* Stats Section */}
+        <FlowRunsStatusSkeleton />
+      </div>
+    </div>
+  );
+}

rnd/autogpt_builder/src/app/marketplace/[id]/page.tsx

@@ -2,7 +2,7 @@ import { Suspense } from "react";
 import { notFound } from "next/navigation";
 import MarketplaceAPI from "@/lib/marketplace-api";
 import { AgentDetailResponse } from "@/lib/marketplace-api";
-import AgentDetailContent from "@/components/AgentDetailContent";
+import AgentDetailContent from "@/components/marketplace/AgentDetailContent";
 
 async function getAgentDetails(id: string): Promise<AgentDetailResponse> {
   const apiUrl =