{ "cells": [ { "cell_type": "markdown", "metadata": { "id": "sddGmEyxMegT" }, "source": [ "\"Open" ] }, { "cell_type": "markdown", "metadata": { "id": "ocOUT0jTNTS6" }, "source": [ "# Run a happy test case" ] }, { "cell_type": "markdown", "metadata": { "id": "35QuPBmFNBL2" }, "source": [ "## Install Dependencies" ] }, { "cell_type": "code", "execution_count": 1, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "-A_XJDB0NAEb", "outputId": "76c23689-306b-4fe8-c9d5-998a1380914f" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Cloning into 'yet-another-applied-llm-benchmark'...\n", "remote: Enumerating objects: 467, done.\u001b[K\n", "remote: Counting objects: 100% (5/5), done.\u001b[K\n", "remote: Compressing objects: 100% (5/5), done.\u001b[K\n", "remote: Total 467 (delta 0), reused 1 (delta 0), pack-reused 462\u001b[K\n", "Receiving objects: 100% (467/467), 170.46 KiB | 1.23 MiB/s, done.\n", "Resolving deltas: 100% (319/319), done.\n", "/content/yet-another-applied-llm-benchmark\n", "Collecting docker (from -r requirements.txt (line 1))\n", " Downloading docker-7.0.0-py3-none-any.whl (147 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m147.6/147.6 kB\u001b[0m \u001b[31m5.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 2)) (1.25.2)\n", "Requirement already satisfied: Pillow in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 3)) (9.4.0)\n", "Collecting openai (from -r requirements.txt (line 4))\n", " Downloading openai-1.12.0-py3-none-any.whl (226 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m226.7/226.7 kB\u001b[0m \u001b[31m14.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: markdown in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 5)) (3.5.2)\n", "Requirement already satisfied: pygments in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 6)) (2.16.1)\n", "Collecting selenium (from -r requirements.txt (line 7))\n", " Downloading selenium-4.18.1-py3-none-any.whl (10.0 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m10.0/10.0 MB\u001b[0m \u001b[31m19.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: packaging>=14.0 in /usr/local/lib/python3.10/dist-packages (from docker->-r requirements.txt (line 1)) (23.2)\n", "Requirement already satisfied: requests>=2.26.0 in /usr/local/lib/python3.10/dist-packages (from docker->-r requirements.txt (line 1)) (2.31.0)\n", "Requirement already satisfied: urllib3>=1.26.0 in /usr/local/lib/python3.10/dist-packages (from docker->-r requirements.txt (line 1)) (2.0.7)\n", "Requirement already satisfied: anyio<5,>=3.5.0 in /usr/local/lib/python3.10/dist-packages (from openai->-r requirements.txt (line 4)) (3.7.1)\n", "Requirement already satisfied: distro<2,>=1.7.0 in /usr/lib/python3/dist-packages (from openai->-r requirements.txt (line 4)) (1.7.0)\n", "Collecting httpx<1,>=0.23.0 (from openai->-r requirements.txt (line 4))\n", " Downloading httpx-0.26.0-py3-none-any.whl (75 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.9/75.9 kB\u001b[0m \u001b[31m13.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: pydantic<3,>=1.9.0 in /usr/local/lib/python3.10/dist-packages (from openai->-r requirements.txt (line 4)) (2.6.1)\n", "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from openai->-r requirements.txt (line 4)) (1.3.0)\n", "Requirement already satisfied: tqdm>4 in /usr/local/lib/python3.10/dist-packages (from openai->-r requirements.txt (line 4)) (4.66.2)\n", "Requirement already satisfied: typing-extensions<5,>=4.7 in /usr/local/lib/python3.10/dist-packages (from openai->-r requirements.txt (line 4)) (4.9.0)\n", "Collecting trio~=0.17 (from selenium->-r requirements.txt (line 7))\n", " Downloading trio-0.24.0-py3-none-any.whl (460 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m460.2/460.2 kB\u001b[0m \u001b[31m19.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting trio-websocket~=0.9 (from selenium->-r requirements.txt (line 7))\n", " Downloading trio_websocket-0.11.1-py3-none-any.whl (17 kB)\n", "Requirement already satisfied: certifi>=2021.10.8 in /usr/local/lib/python3.10/dist-packages (from selenium->-r requirements.txt (line 7)) (2024.2.2)\n", "Requirement already satisfied: idna>=2.8 in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai->-r requirements.txt (line 4)) (3.6)\n", "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai->-r requirements.txt (line 4)) (1.2.0)\n", "Collecting httpcore==1.* (from httpx<1,>=0.23.0->openai->-r requirements.txt (line 4))\n", " Downloading httpcore-1.0.3-py3-none-any.whl (77 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.0/77.0 kB\u001b[0m \u001b[31m13.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting h11<0.15,>=0.13 (from httpcore==1.*->httpx<1,>=0.23.0->openai->-r requirements.txt (line 4))\n", " Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m10.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->openai->-r requirements.txt (line 4)) (0.6.0)\n", "Requirement already satisfied: pydantic-core==2.16.2 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->openai->-r requirements.txt (line 4)) (2.16.2)\n", "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.26.0->docker->-r requirements.txt (line 1)) (3.3.2)\n", "Requirement already satisfied: attrs>=20.1.0 in /usr/local/lib/python3.10/dist-packages (from trio~=0.17->selenium->-r requirements.txt (line 7)) (23.2.0)\n", "Requirement already satisfied: sortedcontainers in /usr/local/lib/python3.10/dist-packages (from trio~=0.17->selenium->-r requirements.txt (line 7)) (2.4.0)\n", "Collecting outcome (from trio~=0.17->selenium->-r requirements.txt (line 7))\n", " Downloading outcome-1.3.0.post0-py2.py3-none-any.whl (10 kB)\n", "Collecting wsproto>=0.14 (from trio-websocket~=0.9->selenium->-r requirements.txt (line 7))\n", " Downloading wsproto-1.2.0-py3-none-any.whl (24 kB)\n", "Requirement already satisfied: pysocks!=1.5.7,<2.0,>=1.5.6 in /usr/local/lib/python3.10/dist-packages (from urllib3>=1.26.0->docker->-r requirements.txt (line 1)) (1.7.1)\n", "Installing collected packages: outcome, h11, wsproto, trio, httpcore, docker, trio-websocket, httpx, selenium, openai\n", "Successfully installed docker-7.0.0 h11-0.14.0 httpcore-1.0.3 httpx-0.26.0 openai-1.12.0 outcome-1.3.0.post0 selenium-4.18.1 trio-0.24.0 trio-websocket-0.11.1 wsproto-1.2.0\n", "Collecting llama-cpp-python (from -r requirements-extra.txt (line 1))\n", " Downloading llama_cpp_python-0.2.44.tar.gz (36.6 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m36.6/36.6 MB\u001b[0m \u001b[31m30.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n", " Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n", " Installing backend dependencies ... \u001b[?25l\u001b[?25hdone\n", " Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", "Collecting anthropic (from -r requirements-extra.txt (line 2))\n", " Downloading anthropic-0.16.0-py3-none-any.whl (846 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m846.4/846.4 kB\u001b[0m \u001b[31m57.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: google-cloud-aiplatform in /usr/local/lib/python3.10/dist-packages (from -r requirements-extra.txt (line 3)) (1.42.1)\n", "Collecting cohere (from -r requirements-extra.txt (line 4))\n", " Downloading cohere-4.48-py3-none-any.whl (52 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m52.2/52.2 kB\u001b[0m \u001b[31m180.8 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: typing-extensions>=4.5.0 in /usr/local/lib/python3.10/dist-packages (from llama-cpp-python->-r requirements-extra.txt (line 1)) (4.9.0)\n", "Requirement already satisfied: numpy>=1.20.0 in /usr/local/lib/python3.10/dist-packages (from llama-cpp-python->-r requirements-extra.txt (line 1)) (1.25.2)\n", "Collecting diskcache>=5.6.1 (from llama-cpp-python->-r requirements-extra.txt (line 1))\n", " Downloading diskcache-5.6.3-py3-none-any.whl (45 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m45.5/45.5 kB\u001b[0m \u001b[31m7.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: jinja2>=2.11.3 in /usr/local/lib/python3.10/dist-packages (from llama-cpp-python->-r requirements-extra.txt (line 1)) (3.1.3)\n", "Requirement already satisfied: anyio<5,>=3.5.0 in /usr/local/lib/python3.10/dist-packages (from anthropic->-r requirements-extra.txt (line 2)) (3.7.1)\n", "Requirement already satisfied: distro<2,>=1.7.0 in /usr/lib/python3/dist-packages (from anthropic->-r requirements-extra.txt (line 2)) (1.7.0)\n", "Requirement already satisfied: httpx<1,>=0.23.0 in /usr/local/lib/python3.10/dist-packages (from anthropic->-r requirements-extra.txt (line 2)) (0.26.0)\n", "Requirement already satisfied: pydantic<3,>=1.9.0 in /usr/local/lib/python3.10/dist-packages (from anthropic->-r requirements-extra.txt (line 2)) (2.6.1)\n", "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from anthropic->-r requirements-extra.txt (line 2)) (1.3.0)\n", "Requirement already satisfied: tokenizers>=0.13.0 in /usr/local/lib/python3.10/dist-packages (from anthropic->-r requirements-extra.txt (line 2)) (0.15.2)\n", "Requirement already satisfied: google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,<3.0.0dev,>=1.34.1 in /usr/local/lib/python3.10/dist-packages (from google-cloud-aiplatform->-r requirements-extra.txt (line 3)) (2.11.1)\n", "Requirement already satisfied: google-auth<3.0.0dev,>=2.14.1 in /usr/local/lib/python3.10/dist-packages (from google-cloud-aiplatform->-r requirements-extra.txt (line 3)) (2.27.0)\n", "Requirement already satisfied: proto-plus<2.0.0dev,>=1.22.0 in /usr/local/lib/python3.10/dist-packages (from google-cloud-aiplatform->-r requirements-extra.txt (line 3)) (1.23.0)\n", "Requirement already satisfied: protobuf!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev,>=3.19.5 in /usr/local/lib/python3.10/dist-packages (from google-cloud-aiplatform->-r requirements-extra.txt (line 3)) (3.20.3)\n", "Requirement already satisfied: packaging>=14.3 in /usr/local/lib/python3.10/dist-packages (from google-cloud-aiplatform->-r requirements-extra.txt (line 3)) (23.2)\n", "Requirement already satisfied: google-cloud-storage<3.0.0dev,>=1.32.0 in /usr/local/lib/python3.10/dist-packages (from google-cloud-aiplatform->-r requirements-extra.txt (line 3)) (2.8.0)\n", "Requirement already satisfied: google-cloud-bigquery<4.0.0dev,>=1.15.0 in /usr/local/lib/python3.10/dist-packages (from google-cloud-aiplatform->-r requirements-extra.txt (line 3)) (3.12.0)\n", "Requirement already satisfied: google-cloud-resource-manager<3.0.0dev,>=1.3.3 in /usr/local/lib/python3.10/dist-packages (from google-cloud-aiplatform->-r requirements-extra.txt (line 3)) (1.12.1)\n", "Requirement already satisfied: shapely<3.0.0dev in /usr/local/lib/python3.10/dist-packages (from google-cloud-aiplatform->-r requirements-extra.txt (line 3)) (2.0.3)\n", "Requirement already satisfied: aiohttp<4.0,>=3.0 in /usr/local/lib/python3.10/dist-packages (from cohere->-r requirements-extra.txt (line 4)) (3.9.3)\n", "Collecting backoff<3.0,>=2.0 (from cohere->-r requirements-extra.txt (line 4))\n", " Downloading backoff-2.2.1-py3-none-any.whl (15 kB)\n", "Collecting fastavro<2.0,>=1.8 (from cohere->-r requirements-extra.txt (line 4))\n", " Downloading fastavro-1.9.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.1 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.1/3.1 MB\u001b[0m \u001b[31m110.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting importlib_metadata<7.0,>=6.0 (from cohere->-r requirements-extra.txt (line 4))\n", " Downloading importlib_metadata-6.11.0-py3-none-any.whl (23 kB)\n", "Requirement already satisfied: requests<3.0.0,>=2.25.0 in /usr/local/lib/python3.10/dist-packages (from cohere->-r requirements-extra.txt (line 4)) (2.31.0)\n", "Requirement already satisfied: urllib3<3,>=1.26 in /usr/local/lib/python3.10/dist-packages (from cohere->-r requirements-extra.txt (line 4)) (2.0.7)\n", "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0,>=3.0->cohere->-r requirements-extra.txt (line 4)) (1.3.1)\n", "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0,>=3.0->cohere->-r requirements-extra.txt (line 4)) (23.2.0)\n", "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0,>=3.0->cohere->-r requirements-extra.txt (line 4)) (1.4.1)\n", "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0,>=3.0->cohere->-r requirements-extra.txt (line 4)) (6.0.5)\n", "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0,>=3.0->cohere->-r requirements-extra.txt (line 4)) (1.9.4)\n", "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0,>=3.0->cohere->-r requirements-extra.txt (line 4)) (4.0.3)\n", "Requirement already satisfied: idna>=2.8 in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->anthropic->-r requirements-extra.txt (line 2)) (3.6)\n", "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->anthropic->-r requirements-extra.txt (line 2)) (1.2.0)\n", "Requirement already satisfied: googleapis-common-protos<2.0.dev0,>=1.56.2 in /usr/local/lib/python3.10/dist-packages (from google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,<3.0.0dev,>=1.34.1->google-cloud-aiplatform->-r requirements-extra.txt (line 3)) (1.62.0)\n", "Requirement already satisfied: grpcio<2.0dev,>=1.33.2 in /usr/local/lib/python3.10/dist-packages (from google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,<3.0.0dev,>=1.34.1->google-cloud-aiplatform->-r requirements-extra.txt (line 3)) (1.60.1)\n", "Requirement already satisfied: grpcio-status<2.0.dev0,>=1.33.2 in /usr/local/lib/python3.10/dist-packages (from google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,<3.0.0dev,>=1.34.1->google-cloud-aiplatform->-r requirements-extra.txt (line 3)) (1.48.2)\n", "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from google-auth<3.0.0dev,>=2.14.1->google-cloud-aiplatform->-r requirements-extra.txt (line 3)) (5.3.2)\n", "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.10/dist-packages (from google-auth<3.0.0dev,>=2.14.1->google-cloud-aiplatform->-r requirements-extra.txt (line 3)) (0.3.0)\n", "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.10/dist-packages (from google-auth<3.0.0dev,>=2.14.1->google-cloud-aiplatform->-r requirements-extra.txt (line 3)) (4.9)\n", "Requirement already satisfied: google-cloud-core<3.0.0dev,>=1.6.0 in /usr/local/lib/python3.10/dist-packages (from google-cloud-bigquery<4.0.0dev,>=1.15.0->google-cloud-aiplatform->-r requirements-extra.txt (line 3)) (2.3.3)\n", "Requirement already satisfied: google-resumable-media<3.0dev,>=0.6.0 in /usr/local/lib/python3.10/dist-packages (from google-cloud-bigquery<4.0.0dev,>=1.15.0->google-cloud-aiplatform->-r requirements-extra.txt (line 3)) (2.7.0)\n", "Requirement already satisfied: python-dateutil<3.0dev,>=2.7.2 in /usr/local/lib/python3.10/dist-packages (from google-cloud-bigquery<4.0.0dev,>=1.15.0->google-cloud-aiplatform->-r requirements-extra.txt (line 3)) (2.8.2)\n", "Requirement already satisfied: grpc-google-iam-v1<1.0.0dev,>=0.12.4 in /usr/local/lib/python3.10/dist-packages (from google-cloud-resource-manager<3.0.0dev,>=1.3.3->google-cloud-aiplatform->-r requirements-extra.txt (line 3)) (0.13.0)\n", "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx<1,>=0.23.0->anthropic->-r requirements-extra.txt (line 2)) (2024.2.2)\n", "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.10/dist-packages (from httpx<1,>=0.23.0->anthropic->-r requirements-extra.txt (line 2)) (1.0.3)\n", "Requirement already satisfied: h11<0.15,>=0.13 in /usr/local/lib/python3.10/dist-packages (from httpcore==1.*->httpx<1,>=0.23.0->anthropic->-r requirements-extra.txt (line 2)) (0.14.0)\n", "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.10/dist-packages (from importlib_metadata<7.0,>=6.0->cohere->-r requirements-extra.txt (line 4)) (3.17.0)\n", "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2>=2.11.3->llama-cpp-python->-r requirements-extra.txt (line 1)) (2.1.5)\n", "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->anthropic->-r requirements-extra.txt (line 2)) (0.6.0)\n", "Requirement already satisfied: pydantic-core==2.16.2 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->anthropic->-r requirements-extra.txt (line 2)) (2.16.2)\n", "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.25.0->cohere->-r requirements-extra.txt (line 4)) (3.3.2)\n", "Requirement already satisfied: huggingface_hub<1.0,>=0.16.4 in /usr/local/lib/python3.10/dist-packages (from tokenizers>=0.13.0->anthropic->-r requirements-extra.txt (line 2)) (0.20.3)\n", "Requirement already satisfied: google-crc32c<2.0dev,>=1.0 in /usr/local/lib/python3.10/dist-packages (from google-resumable-media<3.0dev,>=0.6.0->google-cloud-bigquery<4.0.0dev,>=1.15.0->google-cloud-aiplatform->-r requirements-extra.txt (line 3)) (1.5.0)\n", "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface_hub<1.0,>=0.16.4->tokenizers>=0.13.0->anthropic->-r requirements-extra.txt (line 2)) (3.13.1)\n", "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface_hub<1.0,>=0.16.4->tokenizers>=0.13.0->anthropic->-r requirements-extra.txt (line 2)) (2023.6.0)\n", "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.10/dist-packages (from huggingface_hub<1.0,>=0.16.4->tokenizers>=0.13.0->anthropic->-r requirements-extra.txt (line 2)) (4.66.2)\n", "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from huggingface_hub<1.0,>=0.16.4->tokenizers>=0.13.0->anthropic->-r requirements-extra.txt (line 2)) (6.0.1)\n", "Requirement already satisfied: pyasn1<0.6.0,>=0.4.6 in /usr/local/lib/python3.10/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3.0.0dev,>=2.14.1->google-cloud-aiplatform->-r requirements-extra.txt (line 3)) (0.5.1)\n", "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil<3.0dev,>=2.7.2->google-cloud-bigquery<4.0.0dev,>=1.15.0->google-cloud-aiplatform->-r requirements-extra.txt (line 3)) (1.16.0)\n", "Building wheels for collected packages: llama-cpp-python\n", " Building wheel for llama-cpp-python (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", " Created wheel for llama-cpp-python: filename=llama_cpp_python-0.2.44-cp310-cp310-manylinux_2_35_x86_64.whl size=2590691 sha256=d5974cc437ebeb180349ab448a922c8065407c9c25d347831042ded8973e9a9d\n", " Stored in directory: /root/.cache/pip/wheels/6e/f0/52/1716aa7fefc7eb2a9b76775b0a61fc131b7dcc961e310a048a\n", "Successfully built llama-cpp-python\n", "Installing collected packages: importlib_metadata, fastavro, diskcache, backoff, llama-cpp-python, cohere, anthropic\n", " Attempting uninstall: importlib_metadata\n", " Found existing installation: importlib-metadata 7.0.1\n", " Uninstalling importlib-metadata-7.0.1:\n", " Successfully uninstalled importlib-metadata-7.0.1\n", "Successfully installed anthropic-0.16.0 backoff-2.2.1 cohere-4.48 diskcache-5.6.3 fastavro-1.9.4 importlib_metadata-6.11.0 llama-cpp-python-0.2.44\n", "Traceback (most recent call last):\n", " File \"/content/yet-another-applied-llm-benchmark/tests/print_hello.py\", line 1, in \n", " from evaluator import *\n", " File \"/content/yet-another-applied-llm-benchmark/evaluator.py\", line 31, in \n", " import docker_controller\n", " File \"/content/yet-another-applied-llm-benchmark/docker_controller.py\", line 41, in \n", " BACKEND = json.load(open(\"config.json\"))['container']\n", "FileNotFoundError: [Errno 2] No such file or directory: 'config.json'\n" ] } ], "source": [ "!git clone https://github.com/carlini/yet-another-applied-llm-benchmark.git\n", "# !git clone https://github.com/ViswanathaReddyGajjala/yet-another-applied-llm-benchmark.git\n", "\n", "%cd yet-another-applied-llm-benchmark\n", "!pip install -r requirements.txt\n", "!pip install -r requirements-extra.txt\n" ] }, { "cell_type": "code", "execution_count": 17, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "CJ44Re4nSoTd", "outputId": "d880a595-79eb-42d9-d0f5-00dd2459c44e" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Reading package lists... Done\n", "Building dependency tree... Done\n", "Reading state information... Done\n", "The following package was automatically installed and is no longer required:\n", " libfuse2\n", "Use 'sudo apt autoremove' to remove it.\n", "The following additional packages will be installed:\n", " buildah catatonit conmon containernetworking-plugins crun dns-root-data\n", " dnsmasq-base fuse-overlayfs fuse3 glib-networking glib-networking-common\n", " glib-networking-services golang-github-containernetworking-plugin-dnsname\n", " golang-github-containers-common golang-github-containers-image\n", " gsettings-desktop-schemas iptables libavahi-glib1 libfuse3-3 libgpgme11\n", " libip6tc2 libnetfilter-conntrack3 libnfnetlink0 libnftnl11 libostree-1-1\n", " libproxy1v5 libslirp0 libsoup2.4-1 libsoup2.4-common libyajl2 netbase\n", " session-migration slirp4netns uidmap\n", "Suggested packages:\n", " containers-storage firewalld nftables docker-compose\n", "The following packages will be REMOVED:\n", " fuse\n", "The following NEW packages will be installed:\n", " buildah catatonit conmon containernetworking-plugins crun dns-root-data\n", " dnsmasq-base fuse-overlayfs fuse3 glib-networking glib-networking-common\n", " glib-networking-services golang-github-containernetworking-plugin-dnsname\n", " golang-github-containers-common golang-github-containers-image\n", " gsettings-desktop-schemas iptables libavahi-glib1 libfuse3-3 libgpgme11\n", " libip6tc2 libnetfilter-conntrack3 libnfnetlink0 libnftnl11 libostree-1-1\n", " libproxy1v5 libslirp0 libsoup2.4-1 libsoup2.4-common libyajl2 netbase podman\n", " session-migration slirp4netns uidmap\n", "0 upgraded, 35 newly installed, 1 to remove and 35 not upgraded.\n", "Need to get 27.1 MB of archives.\n", "After this operation, 116 MB of additional disk space will be used.\n", "Get:1 http://archive.ubuntu.com/ubuntu jammy/main amd64 netbase all 6.3 [12.9 kB]\n", "Get:2 http://archive.ubuntu.com/ubuntu jammy/main amd64 libfuse3-3 amd64 3.10.5-1build1 [81.2 kB]\n", "Get:3 http://archive.ubuntu.com/ubuntu jammy/main amd64 fuse3 amd64 3.10.5-1build1 [24.7 kB]\n", "Get:4 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libip6tc2 amd64 1.8.7-1ubuntu5.2 [20.3 kB]\n", "Get:5 http://archive.ubuntu.com/ubuntu jammy/main amd64 libnfnetlink0 amd64 1.0.1-3build3 [14.6 kB]\n", "Get:6 http://archive.ubuntu.com/ubuntu jammy/main amd64 libnetfilter-conntrack3 amd64 1.0.9-1 [45.3 kB]\n", "Get:7 http://archive.ubuntu.com/ubuntu jammy/main amd64 libnftnl11 amd64 1.2.1-1build1 [65.5 kB]\n", "Get:8 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 iptables amd64 1.8.7-1ubuntu5.2 [455 kB]\n", "Get:9 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 uidmap amd64 1:4.8.1-2ubuntu2.2 [22.4 kB]\n", "Get:10 http://archive.ubuntu.com/ubuntu jammy/universe amd64 golang-github-containers-image all 5.16.0-3 [29.3 kB]\n", "Get:11 http://archive.ubuntu.com/ubuntu jammy/universe amd64 golang-github-containers-common all 0.44.4+ds1-1 [28.1 kB]\n", "Get:12 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libgpgme11 amd64 1.16.0-1.2ubuntu4.1 [136 kB]\n", "Get:13 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libavahi-glib1 amd64 0.8-5ubuntu5.2 [8,296 B]\n", "Get:14 http://archive.ubuntu.com/ubuntu jammy/main amd64 libproxy1v5 amd64 0.4.17-2 [51.9 kB]\n", "Get:15 http://archive.ubuntu.com/ubuntu jammy/main amd64 glib-networking-common all 2.72.0-1 [3,718 B]\n", "Get:16 http://archive.ubuntu.com/ubuntu jammy/main amd64 glib-networking-services amd64 2.72.0-1 [9,982 B]\n", "Get:17 http://archive.ubuntu.com/ubuntu jammy/main amd64 session-migration amd64 0.3.6 [9,774 B]\n", "Get:18 http://archive.ubuntu.com/ubuntu jammy/main amd64 gsettings-desktop-schemas all 42.0-1ubuntu1 [31.1 kB]\n", "Get:19 http://archive.ubuntu.com/ubuntu jammy/main amd64 glib-networking amd64 2.72.0-1 [69.8 kB]\n", "Get:20 http://archive.ubuntu.com/ubuntu jammy/main amd64 libsoup2.4-common all 2.74.2-3 [4,008 B]\n", "Get:21 http://archive.ubuntu.com/ubuntu jammy/main amd64 libsoup2.4-1 amd64 2.74.2-3 [287 kB]\n", "Get:22 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libostree-1-1 amd64 2022.2-3 [333 kB]\n", "Get:23 http://archive.ubuntu.com/ubuntu jammy/universe amd64 buildah amd64 1.23.1+ds1-2 [6,094 kB]\n", "Get:24 http://archive.ubuntu.com/ubuntu jammy/universe amd64 catatonit amd64 0.1.7-1 [307 kB]\n", "Get:25 http://archive.ubuntu.com/ubuntu jammy/universe amd64 conmon amd64 2.0.25+ds1-1.1 [35.1 kB]\n", "Get:26 http://archive.ubuntu.com/ubuntu jammy/universe amd64 containernetworking-plugins amd64 0.9.1+ds1-1 [6,422 kB]\n", "Get:27 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libyajl2 amd64 2.1.0-3ubuntu0.22.04.1 [21.0 kB]\n", "Get:28 http://archive.ubuntu.com/ubuntu jammy/universe amd64 crun amd64 0.17+dfsg-1.1 [300 kB]\n", "Get:29 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 dns-root-data all 2023112702~ubuntu0.22.04.1 [5,136 B]\n", "Get:30 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 dnsmasq-base amd64 2.86-1.1ubuntu0.5 [355 kB]\n", "Get:31 http://archive.ubuntu.com/ubuntu jammy/universe amd64 fuse-overlayfs amd64 1.7.1-1 [44.7 kB]\n", "Get:32 http://archive.ubuntu.com/ubuntu jammy/universe amd64 golang-github-containernetworking-plugin-dnsname amd64 1.3.1+ds1-2 [1,083 kB]\n", "Get:33 http://archive.ubuntu.com/ubuntu jammy/main amd64 libslirp0 amd64 4.6.1-1build1 [61.5 kB]\n", "Get:34 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 podman amd64 3.4.4+ds1-1ubuntu1.22.04.2 [10.6 MB]\n", "Get:35 http://archive.ubuntu.com/ubuntu jammy/universe amd64 slirp4netns amd64 1.0.1-2 [28.2 kB]\n", "Fetched 27.1 MB in 2s (17.8 MB/s)\n", "debconf: unable to initialize frontend: Dialog\n", "debconf: (No usable dialog-like program is installed, so the dialog based frontend cannot be used. at /usr/share/perl5/Debconf/FrontEnd/Dialog.pm line 78, <> line 35.)\n", "debconf: falling back to frontend: Readline\n", "debconf: unable to initialize frontend: Readline\n", "debconf: (This frontend requires a controlling tty.)\n", "debconf: falling back to frontend: Teletype\n", "dpkg-preconfigure: unable to re-open stdin: \n", "(Reading database ... 121749 files and directories currently installed.)\n", "Removing fuse (2.9.9-5ubuntu3) ...\n", "Selecting previously unselected package netbase.\n", "(Reading database ... 121739 files and directories currently installed.)\n", "Preparing to unpack .../00-netbase_6.3_all.deb ...\n", "Unpacking netbase (6.3) ...\n", "Selecting previously unselected package libfuse3-3:amd64.\n", "Preparing to unpack .../01-libfuse3-3_3.10.5-1build1_amd64.deb ...\n", "Unpacking libfuse3-3:amd64 (3.10.5-1build1) ...\n", "Selecting previously unselected package fuse3.\n", "Preparing to unpack .../02-fuse3_3.10.5-1build1_amd64.deb ...\n", "Unpacking fuse3 (3.10.5-1build1) ...\n", "Selecting previously unselected package libip6tc2:amd64.\n", "Preparing to unpack .../03-libip6tc2_1.8.7-1ubuntu5.2_amd64.deb ...\n", "Unpacking libip6tc2:amd64 (1.8.7-1ubuntu5.2) ...\n", "Selecting previously unselected package libnfnetlink0:amd64.\n", "Preparing to unpack .../04-libnfnetlink0_1.0.1-3build3_amd64.deb ...\n", "Unpacking libnfnetlink0:amd64 (1.0.1-3build3) ...\n", "Selecting previously unselected package libnetfilter-conntrack3:amd64.\n", "Preparing to unpack .../05-libnetfilter-conntrack3_1.0.9-1_amd64.deb ...\n", "Unpacking libnetfilter-conntrack3:amd64 (1.0.9-1) ...\n", "Selecting previously unselected package libnftnl11:amd64.\n", "Preparing to unpack .../06-libnftnl11_1.2.1-1build1_amd64.deb ...\n", "Unpacking libnftnl11:amd64 (1.2.1-1build1) ...\n", "Selecting previously unselected package iptables.\n", "Preparing to unpack .../07-iptables_1.8.7-1ubuntu5.2_amd64.deb ...\n", "Unpacking iptables (1.8.7-1ubuntu5.2) ...\n", "Selecting previously unselected package uidmap.\n", "Preparing to unpack .../08-uidmap_1%3a4.8.1-2ubuntu2.2_amd64.deb ...\n", "Unpacking uidmap (1:4.8.1-2ubuntu2.2) ...\n", "Selecting previously unselected package golang-github-containers-image.\n", "Preparing to unpack .../09-golang-github-containers-image_5.16.0-3_all.deb ...\n", "Unpacking golang-github-containers-image (5.16.0-3) ...\n", "Selecting previously unselected package golang-github-containers-common.\n", "Preparing to unpack .../10-golang-github-containers-common_0.44.4+ds1-1_all.deb ...\n", "Unpacking golang-github-containers-common (0.44.4+ds1-1) ...\n", "Selecting previously unselected package libgpgme11:amd64.\n", "Preparing to unpack .../11-libgpgme11_1.16.0-1.2ubuntu4.1_amd64.deb ...\n", "Unpacking libgpgme11:amd64 (1.16.0-1.2ubuntu4.1) ...\n", "Selecting previously unselected package libavahi-glib1:amd64.\n", "Preparing to unpack .../12-libavahi-glib1_0.8-5ubuntu5.2_amd64.deb ...\n", "Unpacking libavahi-glib1:amd64 (0.8-5ubuntu5.2) ...\n", "Selecting previously unselected package libproxy1v5:amd64.\n", "Preparing to unpack .../13-libproxy1v5_0.4.17-2_amd64.deb ...\n", "Unpacking libproxy1v5:amd64 (0.4.17-2) ...\n", "Selecting previously unselected package glib-networking-common.\n", "Preparing to unpack .../14-glib-networking-common_2.72.0-1_all.deb ...\n", "Unpacking glib-networking-common (2.72.0-1) ...\n", "Selecting previously unselected package glib-networking-services.\n", "Preparing to unpack .../15-glib-networking-services_2.72.0-1_amd64.deb ...\n", "Unpacking glib-networking-services (2.72.0-1) ...\n", "Selecting previously unselected package session-migration.\n", "Preparing to unpack .../16-session-migration_0.3.6_amd64.deb ...\n", "Unpacking session-migration (0.3.6) ...\n", "Selecting previously unselected package gsettings-desktop-schemas.\n", "Preparing to unpack .../17-gsettings-desktop-schemas_42.0-1ubuntu1_all.deb ...\n", "Unpacking gsettings-desktop-schemas (42.0-1ubuntu1) ...\n", "Selecting previously unselected package glib-networking:amd64.\n", "Preparing to unpack .../18-glib-networking_2.72.0-1_amd64.deb ...\n", "Unpacking glib-networking:amd64 (2.72.0-1) ...\n", "Selecting previously unselected package libsoup2.4-common.\n", "Preparing to unpack .../19-libsoup2.4-common_2.74.2-3_all.deb ...\n", "Unpacking libsoup2.4-common (2.74.2-3) ...\n", "Selecting previously unselected package libsoup2.4-1:amd64.\n", "Preparing to unpack .../20-libsoup2.4-1_2.74.2-3_amd64.deb ...\n", "Unpacking libsoup2.4-1:amd64 (2.74.2-3) ...\n", "Selecting previously unselected package libostree-1-1:amd64.\n", "Preparing to unpack .../21-libostree-1-1_2022.2-3_amd64.deb ...\n", "Unpacking libostree-1-1:amd64 (2022.2-3) ...\n", "Selecting previously unselected package buildah.\n", "Preparing to unpack .../22-buildah_1.23.1+ds1-2_amd64.deb ...\n", "Unpacking buildah (1.23.1+ds1-2) ...\n", "Selecting previously unselected package catatonit.\n", "Preparing to unpack .../23-catatonit_0.1.7-1_amd64.deb ...\n", "Unpacking catatonit (0.1.7-1) ...\n", "Selecting previously unselected package conmon.\n", "Preparing to unpack .../24-conmon_2.0.25+ds1-1.1_amd64.deb ...\n", "Unpacking conmon (2.0.25+ds1-1.1) ...\n", "Selecting previously unselected package containernetworking-plugins.\n", "Preparing to unpack .../25-containernetworking-plugins_0.9.1+ds1-1_amd64.deb ...\n", "Unpacking containernetworking-plugins (0.9.1+ds1-1) ...\n", "Selecting previously unselected package libyajl2:amd64.\n", "Preparing to unpack .../26-libyajl2_2.1.0-3ubuntu0.22.04.1_amd64.deb ...\n", "Unpacking libyajl2:amd64 (2.1.0-3ubuntu0.22.04.1) ...\n", "Selecting previously unselected package crun.\n", "Preparing to unpack .../27-crun_0.17+dfsg-1.1_amd64.deb ...\n", "Unpacking crun (0.17+dfsg-1.1) ...\n", "Selecting previously unselected package dns-root-data.\n", "Preparing to unpack .../28-dns-root-data_2023112702~ubuntu0.22.04.1_all.deb ...\n", "Unpacking dns-root-data (2023112702~ubuntu0.22.04.1) ...\n", "Selecting previously unselected package dnsmasq-base.\n", "Preparing to unpack .../29-dnsmasq-base_2.86-1.1ubuntu0.5_amd64.deb ...\n", "Unpacking dnsmasq-base (2.86-1.1ubuntu0.5) ...\n", "Selecting previously unselected package fuse-overlayfs.\n", "Preparing to unpack .../30-fuse-overlayfs_1.7.1-1_amd64.deb ...\n", "Unpacking fuse-overlayfs (1.7.1-1) ...\n", "Selecting previously unselected package golang-github-containernetworking-plugin-dnsname.\n", "Preparing to unpack .../31-golang-github-containernetworking-plugin-dnsname_1.3.1+ds1-2_amd64.deb ...\n", "Unpacking golang-github-containernetworking-plugin-dnsname (1.3.1+ds1-2) ...\n", "Selecting previously unselected package libslirp0:amd64.\n", "Preparing to unpack .../32-libslirp0_4.6.1-1build1_amd64.deb ...\n", "Unpacking libslirp0:amd64 (4.6.1-1build1) ...\n", "Selecting previously unselected package podman.\n", "Preparing to unpack .../33-podman_3.4.4+ds1-1ubuntu1.22.04.2_amd64.deb ...\n", "Unpacking podman (3.4.4+ds1-1ubuntu1.22.04.2) ...\n", "Selecting previously unselected package slirp4netns.\n", "Preparing to unpack .../34-slirp4netns_1.0.1-2_amd64.deb ...\n", "Unpacking slirp4netns (1.0.1-2) ...\n", "Setting up session-migration (0.3.6) ...\n", "Created symlink /etc/systemd/user/graphical-session-pre.target.wants/session-migration.service → /usr/lib/systemd/user/session-migration.service.\n", "Setting up libproxy1v5:amd64 (0.4.17-2) ...\n", "Setting up libip6tc2:amd64 (1.8.7-1ubuntu5.2) ...\n", "Setting up uidmap (1:4.8.1-2ubuntu2.2) ...\n", "Setting up libyajl2:amd64 (2.1.0-3ubuntu0.22.04.1) ...\n", "Setting up libnftnl11:amd64 (1.2.1-1build1) ...\n", "Setting up libgpgme11:amd64 (1.16.0-1.2ubuntu4.1) ...\n", "Setting up dns-root-data (2023112702~ubuntu0.22.04.1) ...\n", "Setting up golang-github-containers-image (5.16.0-3) ...\n", "Setting up conmon (2.0.25+ds1-1.1) ...\n", "Setting up catatonit (0.1.7-1) ...\n", "Setting up libfuse3-3:amd64 (3.10.5-1build1) ...\n", "Setting up libsoup2.4-common (2.74.2-3) ...\n", "Setting up libslirp0:amd64 (4.6.1-1build1) ...\n", "Setting up libnfnetlink0:amd64 (1.0.1-3build3) ...\n", "Setting up netbase (6.3) ...\n", "Setting up libavahi-glib1:amd64 (0.8-5ubuntu5.2) ...\n", "Setting up golang-github-containers-common (0.44.4+ds1-1) ...\n", "Setting up glib-networking-common (2.72.0-1) ...\n", "Setting up gsettings-desktop-schemas (42.0-1ubuntu1) ...\n", "Setting up glib-networking-services (2.72.0-1) ...\n", "Setting up slirp4netns (1.0.1-2) ...\n", "Setting up crun (0.17+dfsg-1.1) ...\n", "Setting up fuse3 (3.10.5-1build1) ...\n", "Installing new version of config file /etc/fuse.conf ...\n", "Setting up libnetfilter-conntrack3:amd64 (1.0.9-1) ...\n", "Setting up fuse-overlayfs (1.7.1-1) ...\n", "Setting up iptables (1.8.7-1ubuntu5.2) ...\n", "update-alternatives: using /usr/sbin/iptables-legacy to provide /usr/sbin/iptables (iptables) in auto mode\n", "update-alternatives: using /usr/sbin/ip6tables-legacy to provide /usr/sbin/ip6tables (ip6tables) in auto mode\n", "update-alternatives: using /usr/sbin/iptables-nft to provide /usr/sbin/iptables (iptables) in auto mode\n", "update-alternatives: using /usr/sbin/ip6tables-nft to provide /usr/sbin/ip6tables (ip6tables) in auto mode\n", "update-alternatives: using /usr/sbin/arptables-nft to provide /usr/sbin/arptables (arptables) in auto mode\n", "update-alternatives: using /usr/sbin/ebtables-nft to provide /usr/sbin/ebtables (ebtables) in auto mode\n", "Setting up dnsmasq-base (2.86-1.1ubuntu0.5) ...\n", "Setting up containernetworking-plugins (0.9.1+ds1-1) ...\n", "Setting up golang-github-containernetworking-plugin-dnsname (1.3.1+ds1-2) ...\n", "Setting up podman (3.4.4+ds1-1ubuntu1.22.04.2) ...\n", "Created symlink /etc/systemd/user/default.target.wants/podman.service → /usr/lib/systemd/user/podman.service.\n", "Created symlink /etc/systemd/user/sockets.target.wants/podman.socket → /usr/lib/systemd/user/podman.socket.\n", "Created symlink /etc/systemd/system/default.target.wants/podman.service → /lib/systemd/system/podman.service.\n", "Created symlink /etc/systemd/system/sockets.target.wants/podman.socket → /lib/systemd/system/podman.socket.\n", "Created symlink /etc/systemd/system/default.target.wants/podman-auto-update.service → /lib/systemd/system/podman-auto-update.service.\n", "Created symlink /etc/systemd/system/timers.target.wants/podman-auto-update.timer → /lib/systemd/system/podman-auto-update.timer.\n", "Created symlink /etc/systemd/system/default.target.wants/podman-restart.service → /lib/systemd/system/podman-restart.service.\n", "Processing triggers for libglib2.0-0:amd64 (2.72.4-0ubuntu2.2) ...\n", "Processing triggers for libc-bin (2.35-0ubuntu3.4) ...\n", "/sbin/ldconfig.real: /usr/local/lib/libtbbmalloc_proxy.so.2 is not a symbolic link\n", "\n", "/sbin/ldconfig.real: /usr/local/lib/libtbbbind_2_0.so.3 is not a symbolic link\n", "\n", "/sbin/ldconfig.real: /usr/local/lib/libtbbbind_2_5.so.3 is not a symbolic link\n", "\n", "/sbin/ldconfig.real: /usr/local/lib/libtbbmalloc.so.2 is not a symbolic link\n", "\n", "/sbin/ldconfig.real: /usr/local/lib/libtbbbind.so.3 is not a symbolic link\n", "\n", "/sbin/ldconfig.real: /usr/local/lib/libtbb.so.12 is not a symbolic link\n", "\n", "Processing triggers for man-db (2.10.2-1) ...\n", "Processing triggers for dbus (1.12.20-2ubuntu4.1) ...\n", "Setting up glib-networking:amd64 (2.72.0-1) ...\n", "Setting up libsoup2.4-1:amd64 (2.74.2-3) ...\n", "Setting up libostree-1-1:amd64 (2022.2-3) ...\n", "Setting up buildah (1.23.1+ds1-2) ...\n", "Processing triggers for libc-bin (2.35-0ubuntu3.4) ...\n", "/sbin/ldconfig.real: /usr/local/lib/libtbbmalloc_proxy.so.2 is not a symbolic link\n", "\n", "/sbin/ldconfig.real: /usr/local/lib/libtbbbind_2_0.so.3 is not a symbolic link\n", "\n", "/sbin/ldconfig.real: /usr/local/lib/libtbbbind_2_5.so.3 is not a symbolic link\n", "\n", "/sbin/ldconfig.real: /usr/local/lib/libtbbmalloc.so.2 is not a symbolic link\n", "\n", "/sbin/ldconfig.real: /usr/local/lib/libtbbbind.so.3 is not a symbolic link\n", "\n", "/sbin/ldconfig.real: /usr/local/lib/libtbb.so.12 is not a symbolic link\n", "\n" ] } ], "source": [ "!sudo apt-get install podman # this is not necessary\n" ] }, { "cell_type": "code", "execution_count": 18, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "s9X1rz69SZfw", "outputId": "07346a38-7bce-4e94-a6b1-52efd028705b" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "/usr/bin/podman\n" ] } ], "source": [ "!which podman" ] }, { "cell_type": "markdown", "metadata": { "id": "mTE6bJpJUo4R" }, "source": [ "## Setup" ] }, { "cell_type": "markdown", "metadata": { "id": "K31AsKNWNXuB" }, "source": [ "### Add API keys\n", "\n", "You have to add the api keys of the respective models to run them." ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "5A8OwQ_PNOcK", "outputId": "04858270-98ca-4c59-8b60-c8e5d3a67fd2" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Writing config.json\n" ] } ], "source": [ "%%writefile config.json\n", "{\n", " \"container\": \"podman\",\n", " \"hparams\": {\n", " \"temperature\": 0.7\n", " },\n", " \"llms\": {\n", " \"vertexai\": {\n", " \"project_id\": \"TODO\"\n", " },\n", " \"openai\": {\n", " \"api_key\": \"UPDATETHISVALUEIFNEEDED\"\n", " },\n", " \"mistral\": {\n", " \"api_key\": \"TODO\"\n", " },\n", " \"cohere\": {\n", " \"api_key\": \"TODO\"\n", " },\n", " \"anthropic\": {\n", " \"api_key\": \"TODO\"\n", " }\n", " }\n", "}" ] }, { "cell_type": "markdown", "metadata": { "id": "ON5dzImSOzSz" }, "source": [ "For testing purposes, follow these instructions:\n", "\n", "\n", "We'll be using \"gpt-3.5-turbo\" which can be accessed via free accounts.\n", "\n", "\n", "In `llm.py` update the following variables.\n", "1. Set `llm = LLM(\"gpt-3.5-turbo\")`\n", "2. Set `eval_llm = LLM(\"gpt-3.5-turbo\", override_hparams={'temperature': 0.1})`\n", "\n", "\n", "In `evaluator.py`\n", "\n", "1. Update the variable `PYTHON_ENV = \"python3.11\"` to `PYTHON_ENV = \"python\"`\n", "\n", "\n", "In `docker_controller.py` (if not using `podman` (or) `docker`):\n", "1. Set `I_HAVE_BLIND_FAITH_IN_LLMS_AND_AM_OKAY_WITH_THEM_BRICKING_MY_MACHINE_OR_MAKING_THEM_HALT_AND_CATCH_FIRE` to `True`\n", "\n", "If you prefer running it locally:\n", " `Add the respective Python path in evaluator.py .`\n", "\n", "\n", "These changes will enable you to use \"gpt-3.5-turbo\" for testing." ] }, { "cell_type": "markdown", "metadata": { "id": "W-9U4FNoUe6I" }, "source": [ "## Testing" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Hurray! Everything is set up. Now, let's run a simple test case." ] }, { "cell_type": "code", "execution_count": 21, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "GUC4OCZROW81", "outputId": "bdd738f8-a37d-4333-94f7-a843eb74fa12" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "2024-02-21 01:35:23.331165: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", "2024-02-21 01:35:23.331233: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", "2024-02-21 01:35:23.332954: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", "2024-02-21 01:35:23.341169: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n", "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", "2024-02-21 01:35:24.597293: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n", "gpt-3.5-turbo GETCACHE 'Program:\\n\\n```python\\nprint(\"hello world\")\\n```\\n\\nThis program uses the `print()` function in Python to output the string \"hello world\" to the console.'\n", "# Initial Query\n", "> Write a python program that prints the string \"hello world\" and tell me how it works in a sentence\n", "\n", "# LLM Generation\n", "\n", "## Query\n", "> Write a python program that prints the string \"hello world\" and tell me how it works in a sentence\n", "\n", "## Output\n", "> Program:\n", "> \n", "> ```python\n", "> print(\"hello world\")\n", "> ```\n", "> \n", "> This program uses the `print()` function in Python to output the string \"hello world\" to the console.\n", "\n", "# Extract Code\n", "I extracted the following code from that output:\n", "> ```\n", "> print(\"hello world\")\n", "> ```\n", "\n", "# Run Code Interpreter\n", "Running the following program:\n", "> ```\n", "> print(\"hello world\")\n", "> ```\n", "And got the output:\n", "```\n", "hello world\n", "```\n", "\n", "# Substring Evaluation\n", "Testing if the previous output contains the string `hello world`: True\n", "\n", "True\n" ] } ], "source": [ "!PYTHONPATH='.' python tests/print_hello.py" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "wH1T2iv2SuRh" }, "outputs": [], "source": [] } ], "metadata": { "colab": { "provenance": [] }, "kernelspec": { "display_name": "Python 3", "name": "python3" }, "language_info": { "name": "python" } }, "nbformat": 4, "nbformat_minor": 0 }