diff --git a/.dockerignore b/.dockerignore index 758f84a..e35d14b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1,6 @@ .env.test -*.env \ No newline at end of file +*.env +config.yaml +config +/config.example.yaml +/config.test.example.yaml diff --git a/.env.example b/.env.example deleted file mode 100644 index 0c7cc57..0000000 --- a/.env.example +++ /dev/null @@ -1,2 +0,0 @@ -API_KEY= -TOKENS= \ No newline at end of file diff --git a/.env.test.example b/.env.test.example deleted file mode 100644 index 0d4dd99..0000000 --- a/.env.test.example +++ /dev/null @@ -1,7 +0,0 @@ -BOT_TOKEN= -# chat id where file will be saved -CHAT_ID= -# list of bot tokens separated by commas -BOT_TOKENS= -# chat id where file will be forwarded before downloading -DRAFT_CHAT_ID= \ No newline at end of file diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index aea7d3c..bef4915 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -22,14 +22,14 @@ jobs: go-version: ${{ env.GO_VERSION }} - name: Run Tests run: | - ( - echo BOT_TOKEN=${{ secrets.CI_BOT_TOKEN_1 }} - echo BOT_TOKENS=${{ secrets.CI_BOT_TOKEN_1 }},${{ secrets.CI_BOT_TOKEN_2 }},${{ secrets.CI_BOT_TOKEN_3 }} - echo TOKENS=${{ secrets.CI_BOT_TOKEN_1 }},${{ secrets.CI_BOT_TOKEN_2 }},${{ secrets.CI_BOT_TOKEN_3 }} - echo DRAFT_CHAT_ID=${{ secrets.CI_DRAFT_CHAT_ID }} - echo CHAT_ID=${{ secrets.CI_CHAT_ID }} - ) > .env - cp .env .env.test + mkdir config + echo "tokens:" >./config-test/config.yaml + echo " - "${{ secrets.CI_BOT_TOKEN_1 }}"" >> ./config-test/config.yaml + echo " - "${{ secrets.CI_BOT_TOKEN_2 }}"" >> ./config-test/config.yaml + echo " - "${{ secrets.CI_BOT_TOKEN_3 }}"" >> ./config-test/config.yaml + echo "api_key: "" " >> ./config-test/config.yaml + echo "chat_id: ${{ secrets.CI_CHAT_ID }}" >> ./config-test/config.yaml + echo "draft_chat_id: ${{ secrets.CI_DRAFT_CHAT_ID }}" >> ./config-test/config.yaml make test chmod 777 ./scripts/wait-for-it/wait-for-it.sh make docker-e2etest diff --git a/.gitignore b/.gitignore index a5dda83..94b9f1f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ .idea .env.test -*.env \ No newline at end of file +*.env +config.yaml +config.test.yaml +config \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index cf46d40..470bebb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,17 +14,23 @@ COPY . . RUN go build -o bin/tg-bot-storage ./cmd/main.go -RUN apk add --no-cache ca-certificates +#RUN apk add --no-cache ca-certificates # build image with the binary -FROM scratch +FROM alpine +RUN apk add --no-cache ca-certificates +RUN apk add bash # copy certificate to be able to make https request to telegram -COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ +#COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ + +WORKDIR /app # copy the binary -COPY --from=build /build/bin/tg-bot-storage / +COPY --from=build /build/bin/tg-bot-storage /app/ + +VOLUME /app/config EXPOSE 7000 -ENTRYPOINT ["/tg-bot-storage"] \ No newline at end of file +ENTRYPOINT ["/app/tg-bot-storage"] \ No newline at end of file diff --git a/Makefile b/Makefile index e96d773..37277ce 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ APPNAME=tg-bot-storage ## test: run tests on cmd and pkg files. .PHONY: test test: vet fmt - CI="false" go test ./... + CI="false" go test -v -count=1 ./... ## build: build application binary. .PHONY: build @@ -28,7 +28,7 @@ e2etest: ## docker-e2etest: run e2etests in a docker compose docker-e2etest: - docker-compose -f docker-compose.test.yml up --abort-on-container-exit --exit-code-from e2etests + docker-compose -f docker-compose.test.yml up --force-recreate --abort-on-container-exit --exit-code-from e2etests ## docker-build: build the api docker image .PHONY: docker-build diff --git a/README.md b/README.md index 48603cd..782c6c2 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,30 @@ # Telegram Bot Storage(in development) + ![Build Status](https://github.com/dipandaaser/tg-bot-storage/workflows/CI/badge.svg) [![License](https://img.shields.io/github/license/dipandaaser/tg-bot-storage)](LICENSE) [![Release](https://img.shields.io/github/release/dipandaaser/tg-bot-storage.svg)](https://github.com/dipandaaser/tg-bot-storage/releases/latest) [![GitHub Releases Stats of tg-bot-storage](https://img.shields.io/github/downloads/dipandaaser/tg-bot-storage/total.svg?logo=github)](https://somsubhra.github.io/github-release-stats/?username=dipandaaser&repository=tg-bot-storage) -Telegram Bot Storage is a simple library for storing files in Telegram using bot and by passing limits listed on [telegram bot limits website](https://core.telegram.org/bots/faq#my-bot-is-hitting-limits-how-do-i-avoid-this) +Telegram Bot Storage is a simple library for storing files in Telegram using bot and by passing limits listed +on [telegram bot limits website](https://core.telegram.org/bots/faq#my-bot-is-hitting-limits-how-do-i-avoid-this) ### Development #### Dependencies + - Golang 1.16+ - GNU Make +#### How to run + +Create a `config.yaml` file under the [config](config) folder.Use [config.example.yaml](config.example.yaml) as a +template to fill the file. + +Run the following command: `make run` or `go run ./cmd/main.go` + + #### How to test -Create a `.env.test` file with [.env.test.example](.env.test.example) as a template OR just set env var with the same name. -Run the following command: `make test` \ No newline at end of file +Create a `config.yaml` file under the [config-test](config-test) +folder. Use [config.test.example.yaml](config.example.yaml) as a template to fill the file + +Run the following commands: `make test` and `make docker-e2etest` \ No newline at end of file diff --git a/config-test/config.go b/config-test/config.go new file mode 100644 index 0000000..9f9c7ca --- /dev/null +++ b/config-test/config.go @@ -0,0 +1,36 @@ +// +package config_test + +import ( + "github.com/DipandaAser/tg-bot-storage/internal/config" + "gopkg.in/yaml.v3" + "os" +) + +type Config struct { + config.Config `yaml:",inline"` + //ChatID is the ID of the chat to store message, this is use in some test + ChatID int64 `yaml:"chat_id"` + //DraftChatID is the ID of the chat used as a draft chat when downloading file + DraftChatID int64 `yaml:"draft_chat_id"` +} + +var defaultConfig *Config + +//GetDefaultConfig returns a config with default values from the yaml configFilePath +func GetConfig(configFilePath string) Config { + if defaultConfig == nil { + defaultConfig = &Config{Config: config.Config{Tokens: make([]string, 0)}} + file, err := os.Open(configFilePath) + if err != nil { + return Config{} + } + defer file.Close() + + err = yaml.NewDecoder(file).Decode(defaultConfig) + if err != nil { + return Config{} + } + } + return *defaultConfig +} diff --git a/config.example.yaml b/config.example.yaml new file mode 100644 index 0000000..c2b8ca6 --- /dev/null +++ b/config.example.yaml @@ -0,0 +1,5 @@ +tokens: + - "bot token 1" + - "bot token 2" + - "bot token 3" +api_key: "" diff --git a/config.test.example.yaml b/config.test.example.yaml new file mode 100644 index 0000000..874cbc3 --- /dev/null +++ b/config.test.example.yaml @@ -0,0 +1,9 @@ +tokens: + - "bot token 1" + - "bot token 2" + - "bot token 3" +api_key: "" +#chat_id(int) is the ID of the chat to store message, this is use in some test +chat_id: +#draft_chat_id(int) is the ID of the chat used as a draft chat when downloading file +draft_chat_id: \ No newline at end of file diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 6da9e23..7e988e3 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -2,29 +2,24 @@ version: '3' services: api: - restart: always + restart: on-failure build: context: . dockerfile: Dockerfile - env_file: - - ./.env - environment: - api_key: "12345" ports: - - 7000:7000 + - "7000:7000" extra_hosts: - host.docker.internal:host-gateway + volumes: + - "./config-test:/app/config" e2etests: depends_on: - api image: golang:buster command: /app/scripts/wait-for-it/wait-for-it.sh api:7000 -t 300 -- make -C /app e2etest-compose - env_file: - - ./.env environment: CI: "true" api_host: "http://api:7000/" - api_key: "12345" volumes: - .:/app diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..9f441bc --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,10 @@ +version: '3' + +services: + bot-storage: + image: ghcr.io/dipandaaser/bot-storage:latest + restart: on-failure + ports: + - "7000:7000" + volumes: + - "./config:/app/config" \ No newline at end of file diff --git a/internal/config/config.go b/internal/config/config.go index fcca0b4..d907077 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,28 +1,36 @@ package config import ( - "github.com/joho/godotenv" + "gopkg.in/yaml.v3" "os" - "strings" +) + +const ( + configFilePath = "./config/config.yaml" ) var defaultConfig *Config //Config describes the server configuration type Config struct { - Tokens []string - ApiKey string + Tokens []string `yaml:"tokens"` + ApiKey string `yaml:"api_key"` } -//GetDefaultConfig returns a config with default values and env variables +//GetDefaultConfig returns a config with default values from the yaml configFilePath func GetDefaultConfig() Config { if defaultConfig == nil { - //load postgres env variables - _ = godotenv.Load() + defaultConfig = &Config{Tokens: make([]string, 0)} + // read the config file + file, err := os.Open(configFilePath) + if err != nil { + return Config{} + } + defer file.Close() - defaultConfig = &Config{ - Tokens: strings.Split(os.Getenv("TOKENS"), ","), - ApiKey: os.Getenv("API_KEY"), + err = yaml.NewDecoder(file).Decode(defaultConfig) + if err != nil { + return Config{} } } return *defaultConfig diff --git a/pkg/bot/client_test.go b/pkg/bot/client_test.go index 4651606..3948900 100644 --- a/pkg/bot/client_test.go +++ b/pkg/bot/client_test.go @@ -2,35 +2,27 @@ package bot import ( "bytes" - "github.com/joho/godotenv" + config_test "github.com/DipandaAser/tg-bot-storage/config-test" "io/ioutil" "log" - "os" - "strconv" "strings" "sync" "testing" ) const ( - ENVBOTTOKEN = "BOT_TOKEN" - ENVCHATID = "CHAT_ID" - ENVDRAFTCHATID = "DRAFT_CHAT_ID" + configFilePath = "../../config-test/config.yaml" ) -func init() { - _ = godotenv.Load("../../.env.test") -} - func Test_UploadFileReader(t *testing.T) { - client, err := NewClient(os.Getenv(ENVBOTTOKEN)) + client, err := NewClient(config_test.GetConfig(configFilePath).Tokens[0]) if err != nil { t.Fatal(err) return } t.Run("Upload one file", func(t *testing.T) { - chatId, _ := strconv.ParseInt(os.Getenv(ENVCHATID), 10, 64) + chatId := config_test.GetConfig(configFilePath).ChatID data := bytes.NewReader([]byte("data")) _, err = client.UploadFileReader(chatId, t.Name(), data) if err != nil { @@ -41,14 +33,14 @@ func Test_UploadFileReader(t *testing.T) { } func Test_UploadFileBuffer(t *testing.T) { - client, err := NewClient(os.Getenv(ENVBOTTOKEN)) + client, err := NewClient(config_test.GetConfig(configFilePath).Tokens[0]) if err != nil { t.Fatal(err) return } t.Run("Send one file", func(t *testing.T) { - chatId, _ := strconv.ParseInt(os.Getenv(ENVCHATID), 10, 64) + chatId := config_test.GetConfig(configFilePath).ChatID data := []byte("data") _, err := client.UploadFileBuffer(chatId, t.Name(), data) if err != nil { @@ -59,14 +51,14 @@ func Test_UploadFileBuffer(t *testing.T) { } func Test_DownloadFileReader(t *testing.T) { - client, err := NewClient(os.Getenv(ENVBOTTOKEN)) + client, err := NewClient(config_test.GetConfig(configFilePath).Tokens[0]) if err != nil { t.Fatal(err) return } t.Run("Download one file", func(t *testing.T) { - chatId, _ := strconv.ParseInt(os.Getenv(ENVCHATID), 10, 64) + chatId := config_test.GetConfig(configFilePath).ChatID fileContent := "data" msgIdentifier, err := client.UploadFileReader(chatId, t.Name(), strings.NewReader(fileContent)) if err != nil { @@ -74,7 +66,7 @@ func Test_DownloadFileReader(t *testing.T) { return } - draftChatId, _ := strconv.ParseInt(os.Getenv(ENVDRAFTCHATID), 10, 64) + draftChatId := config_test.GetConfig(configFilePath).DraftChatID result, err := client.DownloadFileReader(msgIdentifier, draftChatId) if err != nil { t.Error(err) @@ -94,7 +86,7 @@ func Test_DownloadFileReader(t *testing.T) { }) t.Run("Stress Multiple Download", func(t *testing.T) { - chatId, _ := strconv.ParseInt(os.Getenv(ENVCHATID), 10, 64) + chatId := config_test.GetConfig(configFilePath).ChatID fileContent := "data" msgIdentifier, err := client.UploadFileReader(chatId, t.Name(), strings.NewReader(fileContent)) if err != nil { @@ -103,7 +95,7 @@ func Test_DownloadFileReader(t *testing.T) { } wg := sync.WaitGroup{} - draftChatId, _ := strconv.ParseInt(os.Getenv(ENVDRAFTCHATID), 10, 64) + draftChatId := config_test.GetConfig(configFilePath).DraftChatID lock := sync.Mutex{} count := 0 total := 5 diff --git a/pkg/manager/manager_test.go b/pkg/manager/manager_test.go index b069d80..85f351e 100644 --- a/pkg/manager/manager_test.go +++ b/pkg/manager/manager_test.go @@ -3,20 +3,15 @@ package manager import ( "bytes" "fmt" + config_test "github.com/DipandaAser/tg-bot-storage/config-test" "github.com/joho/godotenv" "log" - "os" - "strconv" - "strings" "sync" "testing" ) const ( - ENVBOTTOKEN = "BOT_TOKEN" - ENVBOTTOKENS = "BOT_TOKENS" - ENVCHATID = "CHAT_ID" - ENVDRAFTCHATID = "DRAFT_CHAT_ID" + configFilePath = "../../config-test/config.yaml" ) func init() { @@ -25,7 +20,7 @@ func init() { func Test_UploadFileReaderWithOneBot(t *testing.T) { - client, err := NewManager(os.Getenv(ENVBOTTOKEN)) + client, err := NewManager(config_test.GetConfig(configFilePath).Tokens[0]) if err != nil { t.Fatal(err) return @@ -36,7 +31,7 @@ func Test_UploadFileReaderWithOneBot(t *testing.T) { t.Run("Stress Upload Test", func(t *testing.T) { wg := sync.WaitGroup{} lock := sync.Mutex{} - chatId, _ := strconv.ParseInt(os.Getenv(ENVCHATID), 10, 64) + chatId := config_test.GetConfig(configFilePath).ChatID count := 0 total := 5 for i := 0; i < total; i++ { @@ -60,9 +55,7 @@ func Test_UploadFileReaderWithOneBot(t *testing.T) { } func Test_UploadFileReaderWithMultipleBot(t *testing.T) { - - botsTokens := strings.Split(os.Getenv(ENVBOTTOKENS), ",") - client, err := NewManager(botsTokens...) + client, err := NewManager(config_test.GetConfig(configFilePath).Tokens...) if err != nil { t.Fatal(err) return @@ -73,7 +66,7 @@ func Test_UploadFileReaderWithMultipleBot(t *testing.T) { t.Run("Stress Upload Test", func(t *testing.T) { wg := sync.WaitGroup{} lock := sync.Mutex{} - chatId, _ := strconv.ParseInt(os.Getenv(ENVCHATID), 10, 64) + chatId := config_test.GetConfig(configFilePath).ChatID count := 0 total := 5 for i := 0; i < total; i++ { @@ -97,7 +90,7 @@ func Test_UploadFileReaderWithMultipleBot(t *testing.T) { } func Test_DownloadFileReader(t *testing.T) { - client, err := NewManager(strings.Split(os.Getenv(ENVBOTTOKENS), ",")...) + client, err := NewManager(config_test.GetConfig(configFilePath).Tokens...) if err != nil { t.Fatal(err) return @@ -105,14 +98,14 @@ func Test_DownloadFileReader(t *testing.T) { go client.StartUploaderManager() - chatId, _ := strconv.ParseInt(os.Getenv(ENVCHATID), 10, 64) + chatId := config_test.GetConfig(configFilePath).ChatID data := bytes.NewReader([]byte("data")) fileIdentifier, err := client.UploadFileReader(chatId, t.Name(), data) if err != nil { t.Fatal(err) } - draftChatId, _ := strconv.ParseInt(os.Getenv(ENVDRAFTCHATID), 10, 64) + draftChatId := config_test.GetConfig(configFilePath).DraftChatID t.Run("Download one file", func(t *testing.T) { _, err := client.DownloadFileReader(fileIdentifier, draftChatId) diff --git a/pkg/rest-client/client_test.go b/pkg/rest-client/client_test.go index 3d914d0..7fcfd51 100644 --- a/pkg/rest-client/client_test.go +++ b/pkg/rest-client/client_test.go @@ -2,23 +2,22 @@ package rest_client import ( "bytes" + config_test "github.com/DipandaAser/tg-bot-storage/config-test" "github.com/stretchr/testify/assert" "os" - "strconv" "strings" "testing" ) func init() { - if n, err := strconv.ParseInt(os.Getenv("CHAT_ID"), 10, 64); err == nil { - chatId = n - } - if n, err := strconv.ParseInt(os.Getenv("DRAFT_CHAT_ID"), 10, 64); err == nil { - draftChatId = n - } + chatId = config_test.GetConfig(configFilePath).ChatID + draftChatId = config_test.GetConfig(configFilePath).DraftChatID } -const skipMsg = "Skipping testing in non CI environment" +const ( + skipMsg = "Skipping testing in non CI environment" + configFilePath = "../../config-test/config.yaml" +) var ( apiHost = os.Getenv("api_host")