583 introduce full test suite with terraform

.buildkite/pipeline.yml

@@ -4,8 +4,20 @@ steps:
       - ".buildkite/steps/build-image.sh"
       - ".buildkite/steps/upload-image.sh"
 
+  - label: ":terraform: build infrastructure"
+    key: "create_infrastructure"
+    if: &is_fts_enabled |
+        (build.pull_request.base_branch == "develop" && !build.pull_request.draft) ||
+        (build.source == "trigger_job" && build.env("NEON_EVM_FULL_TEST_SUITE") == "true")
+    agents:
+      queue: "testing"
+    command:
+      - ".buildkite/steps/full_test_suite/terraform-build.sh"
+
+
   - wait
 
+
   - label: ":cop::skin-tone-2: deploy check"
     command: ".buildkite/steps/deploy-test.sh"
     timeout: 90
@@ -16,15 +28,51 @@ steps:
       - "solana.log"
       - "measurements.log"
       - "evm_loader.log"
-      - "dbcreation.log"
       - "faucet.log"
       - "airdropper.log"
       - "indexer.log"
 
+
+  - label: ":coverage: full test suite (FTS)"
+    key: "full_test_suite"
+    if: *is_fts_enabled
+    commands:
+      - ".buildkite/steps/full_test_suite/run_full_test_suite.sh"
+    env:
+      FTS_THRESHOLD: 1700
+      FTS_CONTAINER_NAME: fts_${BUILDKITE_BUILD_NUMBER}
+      FTS_IMAGE: neonlabsorg/full_test_suite:develop
+      FTS_USERS_NUMBER: 15
+      FTS_JOBS_NUMBER: 8
+      NETWORK_NAME: full-test-suite-${BUILDKITE_BUILD_NUMBER}
+      NETWORK_ID: 111
+      REQUEST_AMOUNT: 20000
+      USE_FAUCET: true
+    agents:
+       queue: "testing"
+    artifact_paths:
+      - allure-reports.tar.gz
+      - fts_${BUILDKITE_BUILD_NUMBER}.log
+
+
   - wait
 
+
   - label: ":floppy_disk: publish image"
     command: ".buildkite/steps/publish-image.sh"
     if: |
         build.branch =~ /^(master|develop|^ci-.+|v[0-9]+\.[0-9]+\..+)$$/ &&
-        (build.env("EVM_LOADER_BRANCH") == "develop" || build.env("EVM_LOADER_BRANCH") == null)
+        (build.env("NEON_EVM_BRANCH") == "develop" || build.env("NEON_EVM_BRANCH") == null)
+
+  - label: ":terraform: destroy infrastructure"
+    agents:
+      queue: "testing"
+    if: *is_fts_enabled
+    command:
+      - ".buildkite/steps/full_test_suite/terraform-destroy.sh"
+    depends_on:
+      - "full_test_suite"
+      - "create_infrastructure"
+    allow_dependency_failure: true
+    artifact_paths:
+      - ".buildkite/steps/full_test_suite/logs/*"

.buildkite/steps/build-image.sh

@@ -4,16 +4,16 @@ set -euo pipefail
 REVISION=$(git rev-parse HEAD)
 
 set ${SOLANA_REVISION:=v1.8.12-testnet}
-set ${EVM_LOADER_REVISION:=latest}
+set ${NEON_EVM_COMMIT:=latest}
 
 # Refreshing neonlabsorg/solana:latest image is required to run .buildkite/steps/build-image.sh locally
 docker pull neonlabsorg/solana:${SOLANA_REVISION}
 
 # Refreshing neonlabsorg/evm_loader:latest image is required to run .buildkite/steps/build-image.sh locally
-docker pull neonlabsorg/evm_loader:${EVM_LOADER_REVISION}
+docker pull neonlabsorg/evm_loader:${NEON_EVM_COMMIT}
 
 docker build -t neonlabsorg/proxy:${REVISION} \
     --build-arg SOLANA_REVISION=${SOLANA_REVISION} \
-    --build-arg EVM_LOADER_REVISION=${EVM_LOADER_REVISION} \
+    --build-arg NEON_EVM_COMMIT=${NEON_EVM_COMMIT} \
     --build-arg PROXY_REVISION=${REVISION} \
     .

.buildkite/steps/full_test_suite/main.tf

@@ -0,0 +1,194 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 3.0"
+    }
+  }
+
+  backend "s3" {
+    // Must be set from environment
+  }
+}
+
+provider "aws" {
+  region = "us-east-2"
+}
+
+data "aws_vpc" "default" {
+  default = true
+}
+
+data "aws_key_pair" "ci-stands" {
+  key_name = "ci-stands"
+}
+
+data "template_file" "solana_init" {
+  template = file("solana_init.sh")
+
+  vars = {
+    branch = "${var.branch}"
+  }
+}
+
+data "template_file" "proxy_init" {
+  template = file("proxy_init.sh")
+
+  vars = {
+    branch    = "${var.branch}"
+    revision  = "latest"
+    solana_ip = aws_instance.solana.private_ip
+  }
+}
+
+resource "random_id" "test-stand-solana" {
+  byte_length = 4
+  prefix      = "test-stand-solana-"
+}
+
+resource "aws_security_group" "test-stand-solana" {
+  name        = random_id.test-stand-solana.hex
+  description = "set of rules allow incoming traffic from ci test agents for OZ tests"
+  vpc_id      = data.aws_vpc.default.id
+
+  ingress {
+    description = "allow incoming from ci test agent to SOLANA"
+    from_port   = 0
+    to_port     = 65535
+    protocol    = "tcp"
+    cidr_blocks = var.allow_list
+  }
+
+  ingress {
+    description = "allow incoming from world to SOLANA"
+    from_port   = 22
+    to_port     = 22
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  egress {
+    from_port        = 0
+    to_port          = 0
+    protocol         = "-1"
+    cidr_blocks      = ["0.0.0.0/0"]
+    ipv6_cidr_blocks = ["::/0"]
+  }
+
+  tags = {
+    Name    = "${var.branch}-test-stand-solana"
+    purpose = "ci-oz-full-tests"
+  }
+}
+
+resource "random_id" "test-stand-proxy" {
+  byte_length = 4
+  prefix      = "test-stand-proxy-"
+}
+
+resource "aws_security_group" "test-stand-proxy" {
+  name        = random_id.test-stand-proxy.hex
+  description = "set of rules allow incoming traffic from ci test agents for OZ tests"
+  vpc_id      = data.aws_vpc.default.id
+
+  ingress {
+    description = "allow incoming from ci test agent to PROXY"
+    from_port   = 9090
+    to_port     = 9091
+    protocol    = "tcp"
+    cidr_blocks = var.allow_list
+  }
+
+  ingress {
+    description = "allow incoming from ci test agent to FAUCET"
+    from_port   = 3333
+    to_port     = 3334
+    protocol    = "tcp"
+    cidr_blocks = var.allow_list
+  }
+
+  ingress {
+    description = "allow incoming from world to PROXY"
+    from_port   = 22
+    to_port     = 22
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  egress {
+    from_port        = 0
+    to_port          = 0
+    protocol         = "-1"
+    cidr_blocks      = ["0.0.0.0/0"]
+    ipv6_cidr_blocks = ["::/0"]
+  }
+
+  tags = {
+    Name    = "${var.branch}-test-stand-proxy"
+    purpose = "ci-oz-full-tests"
+  }
+}
+
+
+resource "aws_instance" "solana" {
+  instance_type          = var.solana_instance_type
+  ami                    = var.ami
+  key_name               = data.aws_key_pair.ci-stands.key_name
+  vpc_security_group_ids = [aws_security_group.test-stand-solana.id]
+  subnet_id              = var.aws_subnet
+
+  ebs_block_device {
+    device_name = "/dev/sda1"
+    volume_size = 50
+  }
+
+  user_data = data.template_file.solana_init.rendered
+
+  tags = {
+    Name    = "${var.branch}-test-stand-solana"
+    purpose = "ci-oz-full-tests"
+  }
+}
+
+resource "aws_instance" "proxy" {
+  instance_type          = var.proxy_instance_type
+  ami                    = var.ami
+  key_name               = data.aws_key_pair.ci-stands.key_name
+  vpc_security_group_ids = [aws_security_group.test-stand-proxy.id]
+  subnet_id              = var.aws_subnet
+  ebs_block_device {
+    device_name = "/dev/sda1"
+    volume_size = 50
+  }
+
+  tags = {
+    Name    = "${var.branch}-test-stand-proxy"
+    purpose = "ci-oz-full-tests"
+  }
+
+  depends_on = [
+    aws_instance.solana
+  ]
+
+  connection {
+    type        = "ssh"
+    user        = "ubuntu"
+    host        = aws_instance.proxy.public_ip
+    private_key = file("~/.ssh/ci-stands")
+  }
+
+  provisioner "file" {
+    content     = data.template_file.proxy_init.rendered
+    destination = "/tmp/proxy_init.sh"
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      "echo '${aws_instance.solana.private_ip}' > /tmp/solana_host",
+      "chmod a+x /tmp/proxy_init.sh",
+      "sudo /tmp/proxy_init.sh"
+    ]
+  }
+}
+
+

.buildkite/steps/full_test_suite/output.tf

@@ -0,0 +1,11 @@
+output "solana_ip" {
+  value = aws_instance.solana.public_ip
+}
+
+output "proxy_ip" {
+  value = aws_instance.proxy.public_ip
+}
+
+output "branch" {
+  value = var.branch
+}

.buildkite/steps/full_test_suite/proxy_init.sh

@@ -0,0 +1,95 @@
+#!/bin/bash
+
+
+# Install docker
+sudo apt-get remove docker docker-engine docker.io containerd runc
+sudo apt-get update
+sudo apt-get -y install ca-certificates curl gnupg lsb-release
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
+echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+sudo apt-get update
+sudo apt-get -y install docker-ce docker-ce-cli containerd.io
+
+
+# Install docker-compose
+sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+sudo chmod +x /usr/local/bin/docker-compose
+
+
+# Get docker-compose file
+cd /opt
+curl -O https://raw.githubusercontent.com/neonlabsorg/proxy-model.py/${branch}/proxy/docker-compose-test.yml
+
+
+# Set required environment variables
+export REVISION=${revision}
+export SOLANA_URL=http:\/\/${solana_ip}:8899
+
+
+# Generate docker-compose override file
+cat > docker-compose-test.override.yml <<EOF
+version: "3"
+
+services:
+  evm_loader:
+    container_name: evm_loader
+    environment:
+      - SOLANA_URL=$SOLANA_URL
+    networks:
+      - net
+    command: bash -c "create-test-accounts.sh 1 && deploy-evm.sh && /opt/spl-token create-account HPsV9Deocecw3GeZv1FkAPNCBRfuVyfw9MMwjwRe1xaU && /opt/spl-token mint HPsV9Deocecw3GeZv1FkAPNCBRfuVyfw9MMwjwRe1xaU 1000000000 --owner /opt/evm_loader-keypair.json -- HX14J4Pp9CgSbWP13Dtpm8VLJpNxMYffLtRCRGsx7Edv"
+  proxy:
+    environment:
+      - SOLANA_URL=$SOLANA_URL
+    ports:
+      - 9091:9090
+  faucet:
+    image: neonlabsorg/faucet:latest
+    environment:
+      - FAUCET_RPC_BIND=0.0.0.0
+      - FAUCET_RPC_PORT=3333
+      - SOLANA_URL=$SOLANA_URL
+      - NEON_ETH_MAX_AMOUNT=50000
+      - EVM_LOADER=53DfF883gyixYNXnM7s5xhdeyV8mVk9T4i2hGV9vG9io
+      - NEON_TOKEN_MINT=HPsV9Deocecw3GeZv1FkAPNCBRfuVyfw9MMwjwRe1xaU   
+      - FAUCET_RPC_ALLOWED_ORIGINS=["https://neonswap.live"]
+      - FAUCET_WEB3_ENABLE=false
+      - FAUCET_SOLANA_ENABLE=true
+      - NEON_TOKEN_MINT_DECIMALS=9
+      - NEON_OPERATOR_KEYFILE=/opt/faucet/id.json
+      - SOLANA_COMMITMENT=confirmed
+    ports:
+      - 3333:3333
+    entrypoint: /opt/faucet/faucet --config /opt/proxy/faucet.conf run
+  airdropper:
+    environment:
+      - SOLANA_URL=$SOLANA_URL
+  indexer:
+    environment:
+      - SOLANA_URL=$SOLANA_URL
+EOF
+
+
+# Get list of services
+SERVICES=$(docker-compose -f docker-compose-test.yml config --services | grep -v "solana")
+
+
+# Check if Solana is available, max attepts is 100(each for 2 seconds)
+CHECK_COMMAND=`curl $SOLANA_URL -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "method":"getHealth"}'`
+MAX_COUNT=100
+CURRENT_ATTEMPT=1
+while [[ "$CHECK_COMMAND" != "{\"jsonrpc\":\"2.0\",\"result\":\"ok\",\"id\":1}" && $CURRENT_ATTEMPT -gt $MAX_COUNT ]]
+do
+  CHECK_COMMAND=`curl $SOLANA_URL -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "method":"getHealth"}'`
+  echo $CHECK_COMMAND >> /tmp/output.txt
+  echo "attempt: $CURRENT_ATTEMPT"
+  ((CURRENT_ATTEMPT=CURRENT_ATTEMPT+1))
+  sleep 2
+done;
+
+
+# Up all services
+docker-compose -f docker-compose-test.yml -f docker-compose-test.override.yml up -d $SERVICES
+
+# Remove unused(solana is required by evm_loader in docker-compose file)
+docker rm -f solana