New Feature: Removes logic dependency

.circleci/config.yml

@@ -51,6 +51,7 @@ jobs:
       - go/install:
           version: << parameters.go_version >>
       - install_dependencies
+      - install_linter
       - run_tests
       - codecov/upload
   test_nightly:
@@ -65,6 +66,7 @@ jobs:
       - go/install:
           version: << parameters.go_version >>
       - install_dependencies
+      - install_linter
       - run_tests
       - codecov/upload
       - slack/notify: &slack-fail-event
@@ -114,18 +116,22 @@ commands:
             sudo apt update
             sudo apt -y install python3 python3-pip python3-setuptools python3-wheel libboost-math-dev libffi-dev
             pip3 install -r misc/requirements.txt
+            pip3 install e2e_tests/
 
       - run:
           name: sync submodules (go-algorand)
           command: |
             git submodule sync
             git submodule update --init
 
-      - run: echo 'export PATH=$PATH:/usr/local/go/bin' >> $BASH_ENV
+      - run: echo 'export PATH=$PATH:/usr/local/go/bin:$HOME/.local/bin' >> $BASH_ENV
 
+  install_linter:
+    description: Install golangci-lint
+    steps:
       - run:
-          name: Install golint
-          command: go install golang.org/x/lint/golint@latest
+          name: Install golangci-lint
+          command: go install github.com/golangci/golangci-lint/cmd/[email protected]
 
   run_tests:
     steps:
@@ -143,6 +149,7 @@ commands:
       - run: make test-generate
       - run: make fakepackage
       - run: make e2e
+      - run: make e2e-conduit
 
   run_indexer_vs_algod:
     steps:

.codecov.yml

@@ -2,6 +2,11 @@ codecov:
   require_ci_to_pass: no
   branch: develop
 
+ignore:
+  - "idb/mocks"
+  - "idb/dummy"
+  - "util/test"
+
 coverage:
   precision: 2
   round: down

.gitignore

@@ -1,5 +1,6 @@
 # Build artifacts
 cmd/algorand-indexer/algorand-indexer
+cmd/conduit/conduit
 api/.3tmp.json
 api/generate
 tmp/
@@ -28,6 +29,9 @@ _*.json
 
 # Python
 __pycache__
+e2e_tests/dist
+e2e_tests/build
+e2e_tests/*egg-info*
 .venv
 
 # jetbrains IDE
@@ -48,3 +52,6 @@ coverage.txt
 
 # asdf
 .tool-versions
+
+# conduit example
+cmd/conduit/data

.golangci.yml

@@ -7,11 +7,11 @@ linters:
   enable:
     - errcheck
     - gofmt
-    - golint
     - govet
     - ineffassign
     - misspell
     - gci
+    - revive
 
 linters-settings:
   gci:
@@ -30,11 +30,11 @@ severity:
     - linters:
         - errcheck
         - gofmt
-        - golint
         - govet
         - ineffassign
         - misspell
         - gci
+        - revive
       severity: error
 
 issues:

.version

@@ -1 +1 @@
-2.14.2
+2.15.0

Makefile

@@ -22,7 +22,12 @@ COVERPKG := $(shell go list ./...  | grep -v '/cmd/' | egrep -v '(testing|test|m
 # Used for e2e test
 export GO_IMAGE = golang:$(shell go version | cut -d ' ' -f 3 | tail -c +3 )
 
-# This is the default target, build the indexer:
+# This is the default target, build everything:
+all: conduit cmd/algorand-indexer/algorand-indexer go-algorand idb/postgres/internal/schema/setup_postgres_sql.go idb/mocks/IndexerDb.go
+
+conduit: go-algorand
+	go generate ./... && cd cmd/conduit && go build
+
 cmd/algorand-indexer/algorand-indexer: idb/postgres/internal/schema/setup_postgres_sql.go go-algorand
 	cd cmd/algorand-indexer && go build -ldflags="${GOLDFLAGS}"
 
@@ -56,7 +61,7 @@ test: idb/mocks/IndexerDb.go cmd/algorand-indexer/algorand-indexer
 	go test -coverpkg=$(COVERPKG) ./... -coverprofile=coverage.txt -covermode=atomic ${TEST_FLAG}
 
 lint: go-algorand
-	golint -set_exit_status ./...
+	golangci-lint run -c .golangci.yml
 	go vet ./...
 
 fmt:
@@ -68,7 +73,11 @@ integration: cmd/algorand-indexer/algorand-indexer
 	test/postgres_integration_test.sh
 
 e2e: cmd/algorand-indexer/algorand-indexer
-	cd misc && docker-compose build --build-arg GO_IMAGE=${GO_IMAGE} && docker-compose up --exit-code-from e2e
+	cd e2e_tests/docker/indexer/ && docker-compose build --build-arg GO_IMAGE=${GO_IMAGE} && docker-compose up --exit-code-from e2e
+
+e2e-conduit: conduit
+	cd third_party/go-algorand && make install
+	export PATH=$(PATH):$(shell go env GOPATH)/bin; pip3 install e2e_tests/ && e2econduit --s3-source-net ${CI_E2E_FILENAME} --conduit-bin cmd/conduit/conduit
 
 deploy:
 	mule/deploy.sh
@@ -96,9 +105,10 @@ indexer-v-algod: nightly-setup indexer-v-algod-swagger nightly-teardown
 # fetch and update submodule. it's default to latest rel/nightly branch.
 # to use a different branch, update the branch in .gitmodules for CI build,
 # and for local testing, you may checkout a specific branch in the submodule.
-# after submodule is updated, CI_E2E_FILE in circleci/config.yml should also
-# be updated to use a newer artifact. path copied from s3 bucket, s3://algorand-testdata/indexer/e2e4/
+# after submodule is updated, CI_E2E_FILENAME in .circleci/config.yml should
+# also be updated to use a newer artifact. path copied from s3 bucket,
+# s3://algorand-testdata/indexer/e2e4/
 update-submodule:
 	git submodule update --remote
 
-.PHONY: test e2e integration fmt lint deploy sign test-package package fakepackage cmd/algorand-indexer/algorand-indexer idb/mocks/IndexerDb.go go-algorand indexer-v-algod
+.PHONY: all test e2e integration fmt lint deploy sign test-package package fakepackage cmd/algorand-indexer/algorand-indexer idb/mocks/IndexerDb.go go-algorand indexer-v-algod conduit

README.md

@@ -48,11 +48,11 @@ Indexer is part of the [sandbox](https://github.com/algorand/sandbox) private ne
 - Search and filter accounts, transactions, assets, and asset balances with many different parameters.
 - Pagination of results.
 - Enriched transaction and account data:
-    - Confirmation round (block containing the transaction)
-    - Confirmation time
-    - Signature type
-    - Close amounts
-    - Create/delete rounds.
+  - Confirmation round (block containing the transaction)
+  - Confirmation time
+  - Signature type
+  - Close amounts
+  - Create/delete rounds.
 - Human readable field names instead of the space optimized protocol level names.
 
 # Contributing
@@ -74,6 +74,9 @@ There are two primary modes of operation:
 
 In both configurations, a postgres connection string is required. Both DSN and URL formats are supported, [details are available here](https://pkg.go.dev/github.com/jackc/pgx/v4/[email protected]#ParseConfig).
 
+In addition, the indexer uses a data directory that stores data needed for runtime operation and configuration.
+See the [Data Directory documentation](docs/DataDirectory.md) for how to (re-)initialize this directory in case it is lost or needs to be re-created.
+
 ### Database updater
 In this mode, the database will be populated with data fetched from an [Algorand archival node](https://developer.algorand.org/docs/run-a-node/setup/types/#archival-mode). Because every block must be fetched to bootstrap the database, the initial import for a ledger with a long history will take a while. If the daemon is terminated, it will resume processing wherever it left off.
 

accounting/eval_preload.go

@@ -9,6 +9,19 @@ import (
 	"github.com/algorand/go-algorand/protocol"
 )
 
+func isNoOpAssetXfer(stxnad *transactions.SignedTxnWithAD) bool {
+	txn := &stxnad.Txn
+	fields := &txn.AssetTransferTxnFields
+
+	if txn.Type != protocol.AssetTransferTx {
+		return false
+	}
+
+	return (fields.AssetAmount == 0) && (txn.Sender != fields.AssetReceiver) && // not an optin
+		fields.AssetCloseTo.IsZero() && // not a closeout
+		fields.AssetSender.IsZero() // not a clawback
+}
+
 // Add requests for asset and app creators to `assetsReq` and `appsReq` for the given
 // transaction.
 func addToCreatorsRequest(stxnad *transactions.SignedTxnWithAD, assetsReq map[basics.AssetIndex]struct{}, appsReq map[basics.AppIndex]struct{}) {
@@ -22,7 +35,8 @@ func addToCreatorsRequest(stxnad *transactions.SignedTxnWithAD, assetsReq map[ba
 		}
 	case protocol.AssetTransferTx:
 		fields := &txn.AssetTransferTxnFields
-		if fields.XferAsset != 0 {
+
+		if fields.XferAsset != 0 && !isNoOpAssetXfer(stxnad) {
 			assetsReq[fields.XferAsset] = struct{}{}
 		}
 	case protocol.AssetFreezeTx:
@@ -111,7 +125,12 @@ func addToAccountsResourcesRequest(stxnad *transactions.SignedTxnWithAD, assetCr
 			}
 		}
 	case protocol.AssetTransferTx:
+		if isNoOpAssetXfer(stxnad) {
+			break
+		}
+
 		fields := &txn.AssetTransferTxnFields
+
 		creatable := ledger.Creatable{
 			Index: basics.CreatableIndex(fields.XferAsset),
 			Type:  basics.AssetCreatable,

api/Makefile

@@ -5,24 +5,27 @@ all:	generated/v2/types.go generated/v2/routes.go generated/common/types.go gene
 generate:	oapi-codegen all
 
 generated/v2/types.go:	indexer.oas3.yml
-	oapi-codegen -package generated -type-mappings integer=uint64 -generate types -exclude-tags=common -o ./generated/v2/types.go indexer.oas3.yml
+	oapi-codegen -config ./generated/v2/v2_types.yml indexer.oas3.yml
 
 generated/v2/routes.go:	indexer.oas3.yml
-	oapi-codegen -package generated -type-mappings integer=uint64 -generate server,spec -exclude-tags=common -o ./generated/v2/routes.go indexer.oas3.yml
+	oapi-codegen -config ./generated/v2/v2_routes.yml indexer.oas3.yml
 
 generated/common/types.go:	indexer.oas3.yml
-	oapi-codegen -package common -type-mappings integer=uint64 -generate types -include-tags=common -o ./generated/common/types.go indexer.oas3.yml
+	oapi-codegen -config ./generated/common/common_types.yml indexer.oas3.yml
 
 generated/common/routes.go:	indexer.oas3.yml
-	oapi-codegen -package common -type-mappings integer=uint64 -generate server,spec -include-tags=common -o ./generated/common/routes.go indexer.oas3.yml
+	oapi-codegen -config ./generated/common/common_routes.yml indexer.oas3.yml
 
 indexer.oas3.yml:	.3tmp.json
 	python3 jsoncanon.py < .3tmp.json > indexer.oas3.yml
 
 .3tmp.json:	indexer.oas2.json
 	curl -s -X POST "https://converter.swagger.io/api/convert" -H "accept: application/json" -H "Content-Type: application/json" -d @./indexer.oas2.json -o .3tmp.json
 
+clean:
+	rm -rf ./generated/common/types.go ./generated/common/routes.go ./generated/v2/types.go ./generated/v2/routes.go
+
 oapi-codegen:	.PHONY
-	go install "github.com/algorand/oapi-codegen/...@v1.3.7"
+	go install "github.com/algorand/oapi-codegen/...@v1.12.0-algorand.0"
 
 .PHONY: