Engineering Test

cli/load_collection_images.go

@@ -3,9 +3,17 @@ package main
 import (
 	nft_proxy "github.com/alphabatem/nft-proxy"
 	token_metadata "github.com/gagliardetto/metaplex-go/clients/token-metadata"
+	services "github.com/alphabatem/nft-proxy/services"
+	"github.com/gagliardetto/solana-go"
 	"log"
+	"io"
+	"net/http"
+	"os"
 )
 
+var solSvc SolanaService
+var solImgSvc SolanaImageService
+
 type collectionLoader struct {
 	metaWorkerCount  int
 	fileWorkerCount  int
@@ -28,6 +36,12 @@ func main() {
 		mediaIn:          make(chan *nft_proxy.Media),
 	}
 
+	err := solSvc.Start()
+
+	if err != nil {
+		return err
+	}
+
 	l.spawnWorkers()
 
 	//TODO Get collection
@@ -37,12 +51,21 @@ func main() {
 	}
 
 	//TODO Fetch all the mints for that collection
+	err := l.fetchAllMints()
+	if err != nil {
+		panic(err)
+	}
+
 	//TODO Fetch Mints/Hash List
 
 	//TODO Batch into batches of 100
 	//TODO Pass to metaDataIn<-
 
 	//TODO Fetch all the metadata accounts for that collection
+	err := l.fetchAccount()
+	if err != nil {
+		panic(err)
+	}
 	//TODO Fetch all images for the accounts
 	//TODO Fetch Image
 	//TODO Resize Image 500x500
@@ -62,23 +85,140 @@ func (l *collectionLoader) spawnWorkers() {
 }
 
 func (l *collectionLoader) loadCollection() error {
+	log.Fatal("Loading collection...")
+
+	// Get block hash from rpc
+	solHash, err := solSvc.RecentBlockhash()
+
+	if err != nil{
+		return err
+	}
+
+	data, _,err := solSvc.TokenData(solHash)
+
+	if err != nil {
+		return err
+	}
+
+	log.Printf("Collection Loaded: %v", data.Collection)
+
+	for _, metadata := range data {
+		l.metaDataIn <- &token_metadata.Metadata{
+			Key:             metadata.Key,
+			UpdateAuthority: metadata.UpdateAuthority,
+			Mint:            metadata.Mint,
+			Data:            metadata.Data,
+			Collection: 	 metadata.Collection
+		}
+	}
+
 	return nil
 }
 
+func (l *collectionLoader) fetchAllMints() error{
+	var mints []solana.PublicKey
+
+	if l.metaDataIn == nil {
+		return err
+	}
+
+	for metadata := range l.metaDataIn{
+		mints = append(mints, metadata.Mints)
+	}
+
+	log.Printf("Mints: %v",mints)
+
+	return nil
+}
+
+func (l *collectionLoader) fetchAccount() error{
+	for metadata := range l.metaDataIn {
+        err := solImgSvc.FetchAccount(metadata.Key)
+        if err != nil {
+            return fmt.Errorf("failed to fetch account: %w", err)
+        }
+    }
+    return nil
+}
+
 //Fetches the off-chain data from the on-chain account & passes to `fileDataWorker`
 func (l *collectionLoader) metaDataWorker() {
+	for metadata := range l.metaDataIn {
+		assetURI := metadata.Data.Uri
 
+		log.Printf("Fetching metadata from URI: %s", assetURI)
+
+		resp, err := http.Get(assetURI)
+		if err != nil {
+			log.Fatalf("failed to fetch metadata from URI %s: %w", uri, err)
+		}
+		defer resp.Body.Close()
+
+		body, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			log.Fatalf("failed to read response body: %w", err)
+		}
+
+		var assetMetadata nft_proxy.NFTMetadataSimple
+
+		err = json.Unmarshal(body, &metadata)
+		if err != nil {
+			log.Fatalf("failed to unmarshal JSON data: %w", err)
+		}
+
+		l.fileDataIn <- assetMetadata
+	}
 }
 
 //Downloads required files & passes to `mediaWorker`
 func (l *collectionLoader) fileDataWorker() {
-
+	filePath := "./raw_cache/solona/"
+	for assetMetadata := range l.fileDataIn {
+		log.Printf("Downloading file: %s", assetMetadata.URI)
+
+		response, err := http.Get(assetMetadata.URI)
+		if err != nil {
+			log.Fatalf("failed to download file: %v", err)
+		}
+		defer response.Body.Close()
+
+		// Create the local file
+		file, err := os.Create(filepath)
+		if err != nil {
+			log.Fatalf("failed to create file: %v", err)
+		}
+		defer file.Close()
+
+		_, err = io.Copy(file, response.Body)
+		if err != nil {
+			log.Fatalf("failed to save file: %v", err)
+		}
+
+		media := &nft_proxy.Media{
+			MediaUri: assetMetadata.ImageUri,
+		}
+
+		l.mediaIn <- media
+	}	
 }
 
 //Stores media data down to SQL
 func (l *collectionLoader) mediaWorker() {
-	for m := range l.mediaIn {
-		//TODO SAVE TO DB
-		log.Printf("M: %s", m.MediaUri)
+	var db *services.SqliteService
+
+	err := db.Start()
+	if err != nil {
+		log.Fatalf(err)
 	}
-}
+
+	defer db.Shutdown()
+
+    for m := range l.mediaIn {
+        // Error handling for database saving
+        _, err = db.Create(m.MediaURI)
+        if err != nil {
+            log.Printf("Error saving media: %v", err)
+        }
+        log.Printf("M: %s", m.MediaUri)
+    }
+}

cli/reload_hashlist.go

@@ -96,6 +96,7 @@ func reloadRemote(hashes Hashlist) error {
 		_, err := c.Get(fmt.Sprintf("https://api.degencdn.com/v1/nfts/%s/image.jpg", h))
 		if err != nil {
 			log.Printf("Failed media: %s - %s", h, err)
+			return err
 		}
 	}
 	return nil
@@ -107,6 +108,7 @@ func reloadLocally(img *services.SolanaImageService, hashes Hashlist) error {
 		_, err := img.Media(h, true)
 		if err != nil {
 			log.Printf("Failed media: %s - %s", h, err)
+			return err
 		}
 	}
 	return nil

constants.go

@@ -2,9 +2,9 @@ package nft_proxy
 
 import "github.com/gagliardetto/solana-go"
 
-const (
-	BASE64_PREFIX = ";base64,"
-)
+// const (
+// 	BASE64_PREFIX = ";base64,"
+// )
 
 var (
 	METAPLEX_CORE = solana.MustPublicKeyFromBase58("CoREENxT6tW1HoK8ypY1SxRMZTcVPm7R94rH4PZNhX7d")

media.go

@@ -22,7 +22,7 @@ type SolanaMedia struct {
 	Mint            string    `json:"mint" gorm:"uniqueIndex"`
 	MintDecimals    uint8     `json:"decimals"`
 	ImageUri        string    `json:"imageUri"`
-	ImageType       string    `json:"ImageType"`
+	ImageType       string    `json:"imageType"`
 	MediaUri        string    `json:"mediaUri"`
 	MediaType       string    `json:"mediaType"`
 	LocalPath       string    `json:"-"`

nft_file_simple.go

@@ -5,12 +5,12 @@ import "strings"
 type NFTMetadataSimple struct {
 	Name     string `json:"name"`
 	Symbol   string `json:"symbol"`
-	Decimals uint8  `json:"-"`
+	Decimals uint8  `json:"decimals"`
 	//Description          string               `json:"description"`
 	//SellerFeeBasisPoints float64              `json:"seller_fee_basis_points"`
 	Image        string `json:"image"`
-	AnimationURL string `json:"animation_url"`
-	ExternalURL  string `json:"external_url"`
+	AnimationURL string `json:"animationUrl"`
+	ExternalURL  string `json:"externalUrl"`
 	//Collection   NFTCollectionSimple `json:"collection"`
 	Properties NFTPropertiesSimple `json:"properties"`
 	//Attributes           []NFTAttributeSimple `json:"attributes"`
@@ -40,7 +40,7 @@ func (m *NFTMetadataSimple) ImageFile() *NFTFiles {
 }
 
 type NFTFiles struct {
-	URL  string `json:"URL"`
+	URL  string `json:"url"`
 	Type string `json:"type"`
 }
 
@@ -60,7 +60,7 @@ type NFTCreatorSimple struct {
 }
 
 type NFTAttributeSimple struct {
-	TraitType string      `json:"trait_type"`
+	TraitType string      `json:"traitType"`
 	Value     interface{} `json:"value"`
 }
 

service/http.go

@@ -60,8 +60,20 @@ func (svc *HttpService) Configure(ctx *context.Context) error {
 }
 
 func (svc *HttpService) Start() error {
-	svc.imgSvc = svc.Service(IMG_SVC).(*ImageService)
-	svc.statSvc = svc.Service(STAT_SVC).(*StatService)
+	image, err := svc.Service(IMG_SVC).(*ImageService)
+
+	if err != nil {
+		return err
+	}		
+
+	stats, err := svc.Service(STAT_SVC).(*StatService)
+
+	if err != nil {
+		return err
+	}
+
+	svc.imgSvc = image
+	svc.statSvc = stats
 
 	r := gin.Default()
 

service/image.go

@@ -37,31 +37,47 @@ func (svc ImageService) Id() string {
 }
 
 func (svc *ImageService) Start() error {
-	svc.solSvc = svc.Service(SOLANA_IMG_SVC).(*SolanaImageService)
-	svc.sql = svc.Service(SQLITE_SVC).(*SqliteService)
-	svc.resize = svc.Service(RESIZE_SVC).(*ResizeService)
+	solImg , err := svc.Service(SOLANA_IMG_SVC).(*SolanaImageService)
+	if err != nil {
+		return err
+	}
+
+	sqlSvc, err := svc.Service(SQLITE_SVC).(*SqliteService)
+	if err != nil {
+		return err
+	}
+
+	resizeSVC , err := svc.Service(RESIZE_SVC).(*ResizeService)
+	if err != nil {
+		return err
+	}
+
+
+	svc.solSvc = solImg 
+	svc.sql = sqlSvc
+	svc.resize = resizeSVC
 
 	svc.httpMedia = &http.Client{Timeout: 10 * time.Second}
 
 	svc.defaultSize = 720 //Gifs will be half the size
 
 	svc.exemptImages = map[string]struct{}{
-		"2kMpEJCZL8vEDZe7YPLMCS9Y3WKSAMedXBn7xHPvsWvi": {},
-		"7xKXtg2CW87d97TXJSDpbD5jBkheTqA83TZRuJosgAsU": {},
-		"AFbX8oGjGpmVFywbVouvhQSRmiW2aR1mohfahi4Y2AdB": {},
-		"CKfatsPMUf8SkiURsDXs7eK6GWb4Jsd6UDbs7twMCWxo": {},
-		"EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v": {},
-		"Es9vMFrzaCERmJfrF4H2FYD4KCoNkY11McCe8BenwNYB": {},
-		"mSoLzYCxHdYgdzU16g5QSh3i5K3z3KZK7ytfqcJm7So":  {},
-		"So11111111111111111111111111111111111111112":  {},
+		"2kMpEJCZL8vEDZe7YPLMCS9Y3WKSAMedXBn7xHPvsWvi.jpg": {},
+		"7xKXtg2CW87d97TXJSDpbD5jBkheTqA83TZRuJosgAsU.jpg": {},
+		"AFbX8oGjGpmVFywbVouvhQSRmiW2aR1mohfahi4Y2AdB.jpg": {},
+		"CKfatsPMUf8SkiURsDXs7eK6GWb4Jsd6UDbs7twMCWxo.jpg": {},
+		"EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v.jpg": {},
+		"Es9vMFrzaCERmJfrF4H2FYD4KCoNkY11McCe8BenwNYB.jpg": {},
+		"mSoLzYCxHdYgdzU16g5QSh3i5K3z3KZK7ytfqcJm7So.jpg":  {},
+		"So11111111111111111111111111111111111111112.jpg":  {},
 	}
 
 	return nil
 }
 
 func (svc *ImageService) Media(key string, skipCache bool) (*nft_proxy.Media, error) {
 	if svc.IsSolKey(key) {
-		return svc.solSvc.Media(key, skipCache)
+		return svc.solSvc.Media(key, skipCache), nil
 	}
 
 	return nil, errors.New("invalid key")

service/resize.go

@@ -52,9 +52,7 @@ func (svc *ResizeService) Resize(data []byte, out io.Writer, size int) error {
 	switch typ {
 	case "png":
 		return png.Encode(out, dst)
-	case "jpeg":
-		return jpeg.Encode(out, dst, &jpeg.Options{Quality: 100})
-	case "jpg":
+	case "jpeg", "jpg":
 		return jpeg.Encode(out, dst, &jpeg.Options{Quality: 100})
 	default:
 		log.Printf("Unsupported media type (%s) encoding as jpeg", typ)
@@ -68,7 +66,10 @@ func (svc *ResizeService) resizeGif(data []byte, width, height int) (*gif.GIF, e
 		return nil, err
 	}
 
-	if width == 0 {
+	if width == 0 && height == 0 {
+		width = im.Config.Width
+		height = im.Config.Height
+	}else if width == 0 {
 		width = int(im.Config.Width * height / im.Config.Width)
 	} else if height == 0 {
 		height = int(width * im.Config.Height / im.Config.Width)