From d467482d3ebdf2de0dd7e7f51a773e3a8ff4ed99 Mon Sep 17 00:00:00 2001 From: Razzmatazz Date: Fri, 10 Jun 2022 09:11:24 -0500 Subject: [PATCH 01/22] use separate kvs for prod and dev --- datasources/ammo.js | 2 +- datasources/barters.js | 2 +- datasources/crafts.js | 2 +- datasources/hideout-legacy.js | 2 +- datasources/hideout.js | 2 +- datasources/historical-prices.js | 6 +++--- datasources/items.js | 2 +- datasources/maps.js | 2 +- datasources/quests.js | 4 ++-- datasources/tasks.js | 2 +- datasources/trader-inventory.js | 9 ++++----- datasources/traders.js | 2 +- index.js | 2 +- loader.js | 4 ++-- script/ci/wrangler-ci.toml | 6 +++--- utils/worker-kv.js | 2 +- wrangler.toml | 6 +++--- 17 files changed, 28 insertions(+), 29 deletions(-) diff --git a/datasources/ammo.js b/datasources/ammo.js index 241f3ea2..6816ce4a 100644 --- a/datasources/ammo.js +++ b/datasources/ammo.js @@ -3,7 +3,7 @@ const WorkerKV = require('../utils/worker-kv'); class AmmoAPI extends WorkerKV { constructor() { - super('AMMO_DATA'); + super('ammo_data'); } async getList() { diff --git a/datasources/barters.js b/datasources/barters.js index e8dfbb14..14cda806 100644 --- a/datasources/barters.js +++ b/datasources/barters.js @@ -2,7 +2,7 @@ const WorkerKV = require('../utils/worker-kv'); class BartersAPI extends WorkerKV { constructor() { - super('BARTER_DATA_V2'); + super('barter_data'); } async getList() { diff --git a/datasources/crafts.js b/datasources/crafts.js index 5fc8849a..8782471a 100644 --- a/datasources/crafts.js +++ b/datasources/crafts.js @@ -3,7 +3,7 @@ const WorkerKV = require('../utils/worker-kv'); class CraftsAPI extends WorkerKV { constructor(){ - super('CRAFT_DATA_V2'); + super('craft_data'); } async getList() { diff --git a/datasources/hideout-legacy.js b/datasources/hideout-legacy.js index 86fb9a22..f5b7b5a4 100644 --- a/datasources/hideout-legacy.js +++ b/datasources/hideout-legacy.js @@ -8,7 +8,7 @@ class HideoutLegacyAPI { return this.moduleList; } - const hideoutData = await ITEM_DATA.get('HIDEOUT_DATA', 'json'); + const hideoutData = await DATA_CACHE.get('hideout_legacy_data', 'json'); const returnData = []; for(const hideoutModule of hideoutData.data){ diff --git a/datasources/hideout.js b/datasources/hideout.js index b88fd2d5..722df6df 100644 --- a/datasources/hideout.js +++ b/datasources/hideout.js @@ -2,7 +2,7 @@ const WorkerKV = require('../utils/worker-kv'); class HideoutAPI extends WorkerKV { constructor() { - super('HIDEOUT_DATA_V3'); + super('hideout_data'); } async getList(){ diff --git a/datasources/historical-prices.js b/datasources/historical-prices.js index a828d6c2..3fb20a36 100644 --- a/datasources/historical-prices.js +++ b/datasources/historical-prices.js @@ -2,7 +2,7 @@ const WorkerKV = require('../utils/worker-kv'); class historicalPricesAPI extends WorkerKV { constructor() { - super('HISTORICAL_PRICES'); + super('historical_price_data'); } async getByItemId(itemId) { @@ -10,8 +10,8 @@ class historicalPricesAPI extends WorkerKV { if (!this.cache) { return Promise.reject(new Error('Historical prices cache is empty')); } - if (!this.cache[itemId]) return []; - return this.cache[itemId]; + if (!this.cache.data[itemId]) return []; + return this.cache.data[itemId]; } } diff --git a/datasources/items.js b/datasources/items.js index 9bee38ff..b721e089 100644 --- a/datasources/items.js +++ b/datasources/items.js @@ -2,7 +2,7 @@ const WorkerKV = require('../utils/worker-kv'); class ItemsAPI extends WorkerKV { constructor() { - super('ITEM_CACHE_V4'); + super('item_data'); } formatItem(rawItem) { diff --git a/datasources/maps.js b/datasources/maps.js index 4006d20d..db067214 100644 --- a/datasources/maps.js +++ b/datasources/maps.js @@ -2,7 +2,7 @@ const WorkerKV = require('../utils/worker-kv'); class MapAPI extends WorkerKV { constructor() { - super('MAP_DATA_V2'); + super('map_data'); } async getList() { diff --git a/datasources/quests.js b/datasources/quests.js index 0e172613..40a92fe6 100644 --- a/datasources/quests.js +++ b/datasources/quests.js @@ -1,13 +1,13 @@ class QuestsAPI { async getList() { - const quests = await ITEM_DATA.get('QUEST_DATA', 'json'); + const quests = await DATA_CACHE.get('quest_data', 'json'); if(!quests){ return {}; } const returnData = []; - for(const quest of quests){ + for(const quest of quests.data){ const parsedQuestData = { ...quest, requirements: quest.require, diff --git a/datasources/tasks.js b/datasources/tasks.js index 2d6f6e9a..d812e309 100644 --- a/datasources/tasks.js +++ b/datasources/tasks.js @@ -2,7 +2,7 @@ const WorkerKV = require('../utils/worker-kv'); class TasksAPI extends WorkerKV { constructor() { - super('TASK_DATA_V2'); + super('task_data'); } async getList() { diff --git a/datasources/trader-inventory.js b/datasources/trader-inventory.js index d1e4be15..0f7ea0c2 100644 --- a/datasources/trader-inventory.js +++ b/datasources/trader-inventory.js @@ -2,7 +2,7 @@ const WorkerKV = require('../utils/worker-kv'); class TraderInventoryAPI extends WorkerKV { constructor() { - super('TRADER_ITEMS_V2'); + super('trader_price_data'); this.traderCache = false; } @@ -14,8 +14,7 @@ class TraderInventoryAPI extends WorkerKV { try { const traderCache = {}; - for (const id in this.cache) { - const itemOffers = this.cache[id]; + for (const itemOffers of Object.values(this.cache.data)) { for (const offer of itemOffers) { if (!traderCache[offer.vendor.trader_id]) traderCache[offer.vendor.trader_id] = []; traderCache[offer.vendor.trader_id].push(offer); @@ -29,10 +28,10 @@ class TraderInventoryAPI extends WorkerKV { async getByItemId(itemId) { await this.init(); - if(!this.cache[itemId]){ + if(!this.cache.data[itemId]){ return []; } - return this.cache[itemId]; + return this.cache.data[itemId]; } async getPricesForTrader(traderId) { diff --git a/datasources/traders.js b/datasources/traders.js index d7646879..4fea41fb 100644 --- a/datasources/traders.js +++ b/datasources/traders.js @@ -38,7 +38,7 @@ const traderNameIdMap = { class TradersAPI extends WorkerKV { constructor() { - super('TRADER_DATA_V2'); + super('trader_data'); } async getList() { diff --git a/index.js b/index.js index fe3ac59a..25a2b29f 100644 --- a/index.js +++ b/index.js @@ -219,4 +219,4 @@ const handleRequest = async event => { (async () => { initSchema(); dataAPI.init(); -})(); \ No newline at end of file +})(); diff --git a/loader.js b/loader.js index 62fb701e..bd5e444d 100644 --- a/loader.js +++ b/loader.js @@ -21,8 +21,8 @@ if (typeof QUERY_CACHE === 'undefined') { } } -if (typeof ITEM_DATA === 'undefined') { - global.ITEM_DATA = { +if (typeof DATA_CACHE === 'undefined') { + global.DATA_CACHE = { get: async (what) => { console.log(`trying to get ${what}`) diff --git a/script/ci/wrangler-ci.toml b/script/ci/wrangler-ci.toml index 60b8e3f7..51db1835 100644 --- a/script/ci/wrangler-ci.toml +++ b/script/ci/wrangler-ci.toml @@ -7,13 +7,13 @@ workers_dev = true webpack_config = "webpack.config.js" # routes = ["api.tarkov.dev/graphql", "api.tarkov.dev/graphql*", "api.tarkov.dev/___graphql", "api.tarkov.dev/webhook*"] kv-namespaces = [ - { binding = "ITEM_DATA", id = "2973a2dd070e4a348d87084171efe11a", preview_id = "2973a2dd070e4a348d87084171efe11a" }, + { binding = "DATA_CACHE", id = "2e6feba88a9e4097b6d2209191ed4ae5", preview_id = "2e6feba88a9e4097b6d2209191ed4ae5" }, { binding = "QUERY_CACHE", id = "7ae67967cff24ceaa31ca58c5a6da798", preview_id = "7ae67967cff24ceaa31ca58c5a6da798" } ] [env.development] # routes = ["dev.api.tarkov.dev/graphql", "dev.api.tarkov.dev/graphql*", "dev.api.tarkov.dev/___graphql", "dev.api.tarkov.dev/webhook*"] kv-namespaces = [ - { binding = "ITEM_DATA", id = "2973a2dd070e4a348d87084171efe11a", preview_id = "2973a2dd070e4a348d87084171efe11a" }, - { binding = "QUERY_CACHE", id = "7ae67967cff24ceaa31ca58c5a6da798", preview_id = "7ae67967cff24ceaa31ca58c5a6da798" } + { binding = "DATA_CACHE", id = "17fd725f04984e408d4a70b37c817171", preview_id = "17fd725f04984e408d4a70b37c817171" }, + { binding = "QUERY_CACHE", id = "cf717460f7eb40299a0f8f844d157051", preview_id = "cf717460f7eb40299a0f8f844d157051" } ] diff --git a/utils/worker-kv.js b/utils/worker-kv.js index a8d95174..6c9e3b51 100644 --- a/utils/worker-kv.js +++ b/utils/worker-kv.js @@ -45,7 +45,7 @@ class WorkerKV { } this.loading = true; return new Promise((resolve, reject) => { - ITEM_DATA.get(this.kvName, 'json').then(data => { + DATA_CACHE.get(this.kvName, 'json').then(data => { this.cache = data; this.loading = false; //console.log(this.kvName, 'listeners', this.events.listenerCount('loaded')); diff --git a/wrangler.toml b/wrangler.toml index a29b93ad..170ed4ac 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -7,7 +7,7 @@ workers_dev = true webpack_config = "webpack.config.js" routes = ["api.tarkov.dev/graphql", "api.tarkov.dev/graphql*", "api.tarkov.dev/___graphql", "api.tarkov.dev/webhook*"] kv-namespaces = [ - { binding = "ITEM_DATA", id = "2973a2dd070e4a348d87084171efe11a", preview_id = "2973a2dd070e4a348d87084171efe11a" }, + { binding = "DATA_CACHE", id = "2e6feba88a9e4097b6d2209191ed4ae5", preview_id = "2e6feba88a9e4097b6d2209191ed4ae5" }, { binding = "QUERY_CACHE", id = "7ae67967cff24ceaa31ca58c5a6da798", preview_id = "7ae67967cff24ceaa31ca58c5a6da798" } ] vars = { ENVIRONMENT = "production" } @@ -16,7 +16,7 @@ vars = { ENVIRONMENT = "production" } routes = [] #routes = ["dev.api.tarkov.dev/graphql", "dev.api.tarkov.dev/graphql*", "dev.api.tarkov.dev/___graphql", "dev.api.tarkov.dev/webhook*"] kv-namespaces = [ - { binding = "ITEM_DATA", id = "2973a2dd070e4a348d87084171efe11a", preview_id = "2973a2dd070e4a348d87084171efe11a" }, - { binding = "QUERY_CACHE", id = "7ae67967cff24ceaa31ca58c5a6da798", preview_id = "7ae67967cff24ceaa31ca58c5a6da798" } + { binding = "DATA_CACHE", id = "17fd725f04984e408d4a70b37c817171", preview_id = "17fd725f04984e408d4a70b37c817171" }, + { binding = "QUERY_CACHE", id = "cf717460f7eb40299a0f8f844d157051", preview_id = "cf717460f7eb40299a0f8f844d157051" } ] vars = { ENVIRONMENT = "development" } From 84e0d3cf37c1ecc29a14d0716a3fd39908027f55 Mon Sep 17 00:00:00 2001 From: Razzmatazz Date: Sun, 12 Jun 2022 08:18:55 -0500 Subject: [PATCH 02/22] reduce kv usage --- datasources/ammo.js | 15 ------- datasources/hideout.js | 15 +++++++ datasources/index.js | 23 +++++----- datasources/items.js | 13 ++++++ datasources/quests.js | 82 ------------------------------------ datasources/tasks.js | 18 +++++++- index.js | 11 ++--- resolvers/ammoResolver.js | 2 +- resolvers/barterResolver.js | 2 +- resolvers/hideoutResolver.js | 9 +++- resolvers/taskResolver.js | 13 +++++- utils/worker-kv.js | 2 +- 12 files changed, 83 insertions(+), 122 deletions(-) delete mode 100644 datasources/ammo.js delete mode 100644 datasources/quests.js diff --git a/datasources/ammo.js b/datasources/ammo.js deleted file mode 100644 index 6816ce4a..00000000 --- a/datasources/ammo.js +++ /dev/null @@ -1,15 +0,0 @@ -// datasource for ammo -const WorkerKV = require('../utils/worker-kv'); - -class AmmoAPI extends WorkerKV { - constructor() { - super('ammo_data'); - } - - async getList() { - await this.init(); - return this.cache.data; - } -} - -module.exports = AmmoAPI; diff --git a/datasources/hideout.js b/datasources/hideout.js index 722df6df..936f6087 100644 --- a/datasources/hideout.js +++ b/datasources/hideout.js @@ -42,6 +42,21 @@ class HideoutAPI extends WorkerKV { } return Promise.reject(new Error(`No hideout station found with id ${id}`)); } + + async getLegacyList() { + await this.init(); + return this.cache.legacy; + } + + async getLegacyModule(name, level) { + await this.init(); + for (const module of this.cache.legacy) { + if (module.name === name && module.quantity === level) { + return module; + } + } + return Promise.reject(new Error(`No hideout module with id ${id} found`)); + } } module.exports = HideoutAPI; diff --git a/datasources/index.js b/datasources/index.js index c0afc0cb..827c64bb 100644 --- a/datasources/index.js +++ b/datasources/index.js @@ -1,32 +1,32 @@ -const AmmoAPI = require('./ammo'); +//const AmmoAPI = require('./ammo'); const BartersAPI = require('./barters'); const CraftsAPI = require('./crafts'); -const HideoutLegacyAPI = require('./hideout-legacy'); +//const HideoutLegacyAPI = require('./hideout-legacy'); const HideoutAPI = require('./hideout'); const HistoricalPricesAPI = require('./historical-prices'); const ItemsAPI = require('./items'); -const QuestsAPI = require('./quests'); +const MapAPI = require('./maps'); +//const QuestsAPI = require('./quests'); const status = require('./status'); +const TasksAPI = require('./tasks'); const TraderInventoryAPI = require('./trader-inventory'); const TradersAPI = require('./traders'); -const TasksAPI = require('./tasks'); -const MapAPI = require('./maps'); class DataSource { constructor(){ - this.ammo = new AmmoAPI(); + //this.ammo = new AmmoAPI(); this.barter = new BartersAPI(); this.craft = new CraftsAPI(); - this.hideoutLegacy = new HideoutLegacyAPI(); + //this.hideoutLegacy = new HideoutLegacyAPI(); this.hideout = new HideoutAPI(); this.historicalPrice = new HistoricalPricesAPI(); this.item = new ItemsAPI(); - this.quest = new QuestsAPI(); + this.map = new MapAPI(); + //this.quest = new QuestsAPI(); + this.status = status; this.traderInventory = new TraderInventoryAPI(); this.trader = new TradersAPI(); this.task = new TasksAPI(); - this.status = status, - this.map = new MapAPI(); this.initialized = false; this.loading = false; @@ -65,6 +65,9 @@ class DataSource { ]).then(() => { this.initialized = true; this.loading = false; + }).catch(error => { + this.loading = false; + return Promise.reject(error); }); } catch (error) { console.error('error initializing data api', error.stack); diff --git a/datasources/items.js b/datasources/items.js index b721e089..ef717915 100644 --- a/datasources/items.js +++ b/datasources/items.js @@ -348,6 +348,19 @@ class ItemsAPI extends WorkerKV { await this.init(); return this.cache.armorMats[matKey]; } + + async getAmmoList() { + const allAmmo = await this.getItemsByBsgCategoryId('5485a8684bdc2da71d8b4567').then(ammoItems => { + // ignore bb + return ammoItems.filter(item => item.id !== '6241c316234b593b5676b637'); + }); + return allAmmo.map(item => { + return { + ...item, + ...item.properties + }; + }); + } } module.exports = ItemsAPI; diff --git a/datasources/quests.js b/datasources/quests.js deleted file mode 100644 index 40a92fe6..00000000 --- a/datasources/quests.js +++ /dev/null @@ -1,82 +0,0 @@ -class QuestsAPI { - async getList() { - const quests = await DATA_CACHE.get('quest_data', 'json'); - - if(!quests){ - return {}; - } - const returnData = []; - - for(const quest of quests.data){ - const parsedQuestData = { - ...quest, - requirements: quest.require, - wikiLink: quest.wiki, - reputation: quest.reputation.map((reputationData) => { - return { - trader: reputationData.trader, - amount: reputationData.rep, - }; - }), - objectives: quest.objectives.map((objectiveData) => { - const formattedObjective = { - ...objectiveData, - }; - - if(objectiveData.type === 'collect' || objectiveData.type === 'find' || objectiveData.type === 'place'){ - formattedObjective.targetItem = formattedObjective.target; - - if(!formattedObjective.targetItem.id){ - //console.log(`${quest.id} - ${formattedObjective.target}`); - formattedObjective.targetItem = null; - } - } else if (objectiveData.type === 'mark') { - formattedObjective.targetItem = formattedObjective.tool; - - if(!formattedObjective.targetItem.id){ - //console.log(`${quest.id} - ${formattedObjective.tool}`); - formattedObjective.targetItem = null; - } - } - - if(!Array.isArray(formattedObjective.target)){ - formattedObjective.target = [formattedObjective.target]; - } - - return formattedObjective; - }), - }; - - parsedQuestData.requirements.quests = parsedQuestData.requirements.quests.map((stringOrArray) => { - if(Array.isArray(stringOrArray)){ - return stringOrArray; - } - - return [stringOrArray]; - }); - - returnData.push(parsedQuestData); - } - - for(const quest of returnData){ - if(quest.require.quests.length === 0){ - quest.require.prerequisiteQuests = [[]]; - continue; - } - - let questsList = []; - - for(const questList of quest.require.quests){ - questsList.push(questList.map((id) => { - return returnData.find(tempQuest => tempQuest.id === id); - })); - } - - quest.require.prerequisiteQuests = questsList; - } - - return returnData; - } -} - -module.exports = QuestsAPI; diff --git a/datasources/tasks.js b/datasources/tasks.js index d812e309..9757cfdd 100644 --- a/datasources/tasks.js +++ b/datasources/tasks.js @@ -2,7 +2,7 @@ const WorkerKV = require('../utils/worker-kv'); class TasksAPI extends WorkerKV { constructor() { - super('task_data'); + super('quest_data'); } async getList() { @@ -104,6 +104,22 @@ class TasksAPI extends WorkerKV { return rawTask; }); } + + async getQuests() { + await this.init(); + return this.cache.legacy; + } + + async getQuest(id) { + await this.init(); + const quests = await this.getQuests(); + for (const quest of quests) { + if (quest.id === id) { + return quest; + } + } + return Promise.reject(new Error(`No quest with id ${id} found`)); + } } module.exports = TasksAPI; diff --git a/index.js b/index.js index 25a2b29f..71cd4905 100644 --- a/index.js +++ b/index.js @@ -28,7 +28,7 @@ let loadingSchema = false; * Example of how router can be used in an application * */ -async function getSchema() { +async function getSchema(data) { if (schema){ return schema; } @@ -50,7 +50,7 @@ async function getSchema() { }); } loadingSchema = true; - return dynamicTypeDefs(dataAPI).then(dynamicTypeDefs => { + return dynamicTypeDefs(data).then(dynamicTypeDefs => { schema = makeExecutableSchema({typeDefs: mergeTypeDefs([typeDefs, dynamicTypeDefs]), resolvers: resolvers}); loadingSchema = false; //schemaEvents.emit('loaded'); @@ -110,7 +110,7 @@ async function graphqlHandler(event, graphQLOptions) { } */ await dataAPI.init(); - const result = await graphql(await getSchema(), query, {}, {data: dataAPI, util: graphqlUtil}, variables); + const result = await graphql(await getSchema(dataAPI), query, {}, {data: dataAPI, util: graphqlUtil}, variables); const body = JSON.stringify(result); /* if(!result.errors && !url.hostname.includes('localhost') && !url.hostname.includes('tutorial.cloudflareworkers.com')){ @@ -215,8 +215,3 @@ const handleRequest = async event => { return new Response(graphQLOptions.debug ? err : 'Something went wrong', { status: 500 }); } }; - -(async () => { - initSchema(); - dataAPI.init(); -})(); diff --git a/resolvers/ammoResolver.js b/resolvers/ammoResolver.js index b22876c5..7c111cd7 100644 --- a/resolvers/ammoResolver.js +++ b/resolvers/ammoResolver.js @@ -1,7 +1,7 @@ module.exports = { Query: { ammo(obj, args, context, info) { - return context.data.ammo.getList(); + return context.data.item.getAmmoList(); } }, Ammo: { diff --git a/resolvers/barterResolver.js b/resolvers/barterResolver.js index 00f15da9..616cd271 100644 --- a/resolvers/barterResolver.js +++ b/resolvers/barterResolver.js @@ -7,7 +7,7 @@ module.exports = { Barter: { taskUnlock(data, args, context) { if (!data || !data.taskUnlock) return null; - return context.data.task.get(data.taskUnlock); + return context.data.quest.get(data.taskUnlock); }, trader(data, args, context) { return context.data.trader.get(data.trader_id); diff --git a/resolvers/hideoutResolver.js b/resolvers/hideoutResolver.js index e75a6093..d807f8fe 100644 --- a/resolvers/hideoutResolver.js +++ b/resolvers/hideoutResolver.js @@ -1,7 +1,7 @@ module.exports = { Query: { hideoutModules(obj, args, context, info) { - return context.data.hideoutLegacy.getList(); + return context.data.hideout.getLegacyList(); }, hideoutStations(obj, args, context, info) { return context.data.hideout.getList(); @@ -29,5 +29,12 @@ module.exports = { station(data, args, context) { return context.data.hideout.getStation(data.station); } + }, + HideoutModule: { + moduleRequirements(data, args, context) { + return data.moduleRequirements.map(req => { + return context.data.hideout.getLegacyModule(req.name, req.quantity); + }); + } } }; diff --git a/resolvers/taskResolver.js b/resolvers/taskResolver.js index bc40a7f7..915c1e60 100644 --- a/resolvers/taskResolver.js +++ b/resolvers/taskResolver.js @@ -16,7 +16,7 @@ module.exports = { return context.data.task.get(args.id); }, quests(obj, args, context, info) { - return context.data.quest.getList(); + return context.data.task.getQuests(); } }, Task: { @@ -287,9 +287,18 @@ module.exports = { return context.data.item.getItem(data.targetItem) } }, + QuestRequirement: { + prerequisiteQuests(data, args, context) { + return data.prerequisiteQuests.map(questArray => { + return questArray.map(questId => { + return context.data.task.getQuest(questId); + }); + }); + } + }, QuestRewardReputation: { async trader(data, args, context) { - return context.data.trader.getByName(data.trader); + return context.data.trader.get(context.data.trader.getDataIdMap()[data.trader]); } } }; diff --git a/utils/worker-kv.js b/utils/worker-kv.js index 6c9e3b51..2c75604c 100644 --- a/utils/worker-kv.js +++ b/utils/worker-kv.js @@ -12,7 +12,7 @@ class WorkerKV { async init(){ if (this.cache){ //console.log('already loaded', this.kvName); - return true; + return; } if (this.loading) { //console.log('waiting for load', this.kvName); From 7bc9a28da3b2c8a6b25c737459eb7c65925e7382 Mon Sep 17 00:00:00 2001 From: Razzmatazz Date: Tue, 14 Jun 2022 11:33:43 -0500 Subject: [PATCH 03/22] gzip --- .github/workflows/pull_request.yml | 2 +- index.js | 47 ++++++++++++++++++------------ package.json | 2 +- utils/worker-kv.js | 19 +++++++++--- wrangler.toml | 18 ++++++++---- 5 files changed, 59 insertions(+), 29 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 37894510..797f2a79 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -30,7 +30,7 @@ jobs: uses: cloudflare/wrangler-action@3424d15af26edad39d5276be3cc0cc9ffec22b55 # pin@1.3.0 with: apiToken: ${{ secrets.CF_API_TOKEN }} - environment: "development" + environment: "staging" # Post comment on PR with development deploy info - uses: actions/github-script@5d03ada4b0a753e9460b312e61cc4f8fdeacf163 # pin@0.9.0 diff --git a/index.js b/index.js index 71cd4905..a4676da8 100644 --- a/index.js +++ b/index.js @@ -62,18 +62,24 @@ async function getSchema(data) { } addEventListener('fetch', event => { + console.time(event.request.cf.tlsExportedAuthenticator.clientFinished+' total response'); + console.log('environment', ENVIRONMENT); event.respondWith(handleRequest(event)); }); async function graphqlHandler(event, graphQLOptions) { + //console.time(event.request.cf.tlsExportedAuthenticator.clientFinished+' checkPoint'); const request = event.request; const url = new URL(request.url); let query = false; let variables = false; - if(request.method === 'POST'){ + if (request.method === 'POST') { try { + //console.timeEnd(event.request.cf.tlsExportedAuthenticator.clientFinished+' checkPoint'); + console.time(event.request.cf.tlsExportedAuthenticator.clientFinished+' request.json'); const requestBody = await request.json(); + console.timeEnd(event.request.cf.tlsExportedAuthenticator.clientFinished+' request.json'); query = requestBody.query; variables = requestBody.variables; } catch (jsonError){ @@ -83,11 +89,22 @@ async function graphqlHandler(event, graphQLOptions) { status: 503, }); } - } - - if(request.method === 'GET'){ + } else if (request.method === 'GET') { query = url.searchParams.get('query'); variables = url.searchParams.get('variables'); + } else { + return new Response(null, { + status: 501, + }); + } + // Check for empty /graphql query + if (!query || query.trim() === "") { + return new Response('GraphQL requires a query in the body of the request', + { + status: 200, + headers: { 'cache-control': 'public, max-age=2592000' } + } + ); } /* const queryHashString = JSON.stringify({ @@ -109,14 +126,20 @@ async function graphqlHandler(event, graphQLOptions) { } } */ - await dataAPI.init(); + console.time(event.request.cf.tlsExportedAuthenticator.clientFinished+' init') + try { + await dataAPI.init(); + } catch (error) { + console.log('init error', error, error.stack); + } + console.timeEnd(event.request.cf.tlsExportedAuthenticator.clientFinished+' init') const result = await graphql(await getSchema(dataAPI), query, {}, {data: dataAPI, util: graphqlUtil}, variables); const body = JSON.stringify(result); /* if(!result.errors && !url.hostname.includes('localhost') && !url.hostname.includes('tutorial.cloudflareworkers.com')){ await QUERY_CACHE.put(queryHash, body, {expirationTtl: 300}); } */ - + console.timeEnd(event.request.cf.tlsExportedAuthenticator.clientFinished+' total response') return new Response(body, { headers: { 'content-type': 'application/json', @@ -160,18 +183,6 @@ const handleRequest = async event => { const request = event.request; const url = new URL(request.url); - // Check for empty /graphql query - if (url.pathname === "/graphql" && request.method === 'POST') { - const json = await request.clone().json(); - if (json.query.trim() === "") { - // Clone the response so that it's no longer immutable - const response = new Response('GraphQL requires a query in the body of the request', { status: 200 }); - // Add a cache control header - response.headers.append('cache-control', 'public, max-age=2592000'); - return response; - } - } - try { if(url.pathname === '/webhook/nightbot'){ return nightbot(request, dataAPI); diff --git a/package.json b/package.json index 48ea5e46..46e3df3a 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "test": "echo \"Error: no test specified\"", "format": "prettier --write '**/*.{js,css,json,md}'", "dev": "wrangler --env development dev", - "ci": "wrangler --env development --config script/ci/wrangler-ci.toml dev", + "ci": "wrangler --env development dev", "local": "wrangler dev" }, "author": "Oskar Risberg ", diff --git a/utils/worker-kv.js b/utils/worker-kv.js index 2c75604c..aa6ddf26 100644 --- a/utils/worker-kv.js +++ b/utils/worker-kv.js @@ -1,4 +1,5 @@ const {EventEmitter} = require('events'); +const zlib = require('zlib'); class WorkerKV { constructor(kvName){ @@ -44,15 +45,25 @@ class WorkerKV { });*/ } this.loading = true; + console.time('kv load '+this.kvName); return new Promise((resolve, reject) => { - DATA_CACHE.get(this.kvName, 'json').then(data => { - this.cache = data; + DATA_CACHE.getWithMetadata(this.kvName, 'text').then(response => { + const data = response.value; + const metadata = response.metadata; + console.timeEnd('kv load '+this.kvName); + if (metadata && metadata.compression) { + if (metadata.compression = 'gzip') { + this.cache = JSON.parse(zlib.gunzipSync(Buffer.from(data, metadata.encoding)).toString()); + } else { + return reject(new Error(`${metadata.compression} is not a recognized compression type`)); + } + } else { + this.cache = JSON.parse(data); + } this.loading = false; //console.log(this.kvName, 'listeners', this.events.listenerCount('loaded')); try { - //console.time(this.kvName+' emit'); this.events.emit('loaded'); - //console.timeEnd(this.kvName+' emit'); } catch (error) { console.log(error); } diff --git a/wrangler.toml b/wrangler.toml index 170ed4ac..21819d02 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -5,18 +5,26 @@ account_id = "424ad63426a1ae47d559873f929eb9fc" zone_id = "777d475095752ccbe5d1d49e690a305b" workers_dev = true webpack_config = "webpack.config.js" + +[env.production] routes = ["api.tarkov.dev/graphql", "api.tarkov.dev/graphql*", "api.tarkov.dev/___graphql", "api.tarkov.dev/webhook*"] kv-namespaces = [ - { binding = "DATA_CACHE", id = "2e6feba88a9e4097b6d2209191ed4ae5", preview_id = "2e6feba88a9e4097b6d2209191ed4ae5" }, - { binding = "QUERY_CACHE", id = "7ae67967cff24ceaa31ca58c5a6da798", preview_id = "7ae67967cff24ceaa31ca58c5a6da798" } + { binding = "DATA_CACHE", id = "2e6feba88a9e4097b6d2209191ed4ae5", preview_id = "17fd725f04984e408d4a70b37c817171" }, + { binding = "QUERY_CACHE", id = "7ae67967cff24ceaa31ca58c5a6da798", preview_id = "cf717460f7eb40299a0f8f844d157051" } ] vars = { ENVIRONMENT = "production" } -[env.development] -routes = [] -#routes = ["dev.api.tarkov.dev/graphql", "dev.api.tarkov.dev/graphql*", "dev.api.tarkov.dev/___graphql", "dev.api.tarkov.dev/webhook*"] +[env.staging] +routes = ["dev.api.tarkov.dev/graphql", "dev.api.tarkov.dev/graphql*", "dev.api.tarkov.dev/___graphql", "dev.api.tarkov.dev/webhook*"] kv-namespaces = [ { binding = "DATA_CACHE", id = "17fd725f04984e408d4a70b37c817171", preview_id = "17fd725f04984e408d4a70b37c817171" }, { binding = "QUERY_CACHE", id = "cf717460f7eb40299a0f8f844d157051", preview_id = "cf717460f7eb40299a0f8f844d157051" } ] +vars = { ENVIRONMENT = "staging" } + +[env.development] +kv-namespaces = [ + { binding = "DATA_CACHE", id = "2e6feba88a9e4097b6d2209191ed4ae5", preview_id = "17fd725f04984e408d4a70b37c817171" }, + { binding = "QUERY_CACHE", id = "7ae67967cff24ceaa31ca58c5a6da798", preview_id = "cf717460f7eb40299a0f8f844d157051" } +] vars = { ENVIRONMENT = "development" } From 2f5d1cef84ba7aacb93e0c3f740f6ddad903b7e1 Mon Sep 17 00:00:00 2001 From: Razzmatazz Date: Sat, 18 Jun 2022 22:52:44 -0500 Subject: [PATCH 04/22] skip cache on dev --- datasources/index.js | 1 + index.js | 27 +++++++++++++++++---------- utils/worker-kv.js | 16 ++++++++-------- 3 files changed, 26 insertions(+), 18 deletions(-) diff --git a/datasources/index.js b/datasources/index.js index 827c64bb..a481b149 100644 --- a/datasources/index.js +++ b/datasources/index.js @@ -57,6 +57,7 @@ class DataSource { this.barter.init(), this.craft.init(), this.hideout.init(), + this.historicalPrice.init(), this.item.init(), this.map.init(), this.task.init(), diff --git a/index.js b/index.js index 2085fddc..668ae69b 100644 --- a/index.js +++ b/index.js @@ -22,6 +22,7 @@ const twitch = require('./custom-endpoints/twitch'); let schema = false; let loadingSchema = false; +const skipCache = ENVIRONMENT !== 'production' || false; //const schemaEvents = new EventEmitter(); //schemaEvents.setMaxListeners(0); @@ -116,14 +117,18 @@ async function graphqlHandler(event, graphQLOptions) { }; // Check the cache service for data first - If cached data exists, return it - const cachedResponse = await cacheMachine.get(query); - if (cachedResponse) { - // Construct a new response with the cached data - const newResponse = new Response(cachedResponse, headers); - // Add a custom 'X-CACHE: HIT' header so we know the request hit the cache - newResponse.headers.append('X-CACHE', 'HIT'); - // Return the new cached response - return newResponse; + if (!skipCache) { + const cachedResponse = await cacheMachine.get(query); + if (cachedResponse) { + // Construct a new response with the cached data + const newResponse = new Response(cachedResponse, headers); + // Add a custom 'X-CACHE: HIT' header so we know the request hit the cache + newResponse.headers.append('X-CACHE', 'HIT'); + // Return the new cached response + return newResponse; + } + } else { + console.log(`Skipping cache in ${ENVIRONMENT} environment`); } /* const queryHashString = JSON.stringify({ @@ -156,8 +161,10 @@ async function graphqlHandler(event, graphQLOptions) { const body = JSON.stringify(result); // Update the cache with the results of the query - // using waitUntil doens't hold up returning a response but keeps the worker alive as long as needed - event.waitUntil(cacheMachine.put(query, body)); + if (!skipCache) { + // using waitUntil doens't hold up returning a response but keeps the worker alive as long as needed + event.waitUntil(cacheMachine.put(query, body)); + } /* if(!result.errors && !url.hostname.includes('localhost') && !url.hostname.includes('tutorial.cloudflareworkers.com')){ await QUERY_CACHE.put(queryHash, body, {expirationTtl: 300}); diff --git a/utils/worker-kv.js b/utils/worker-kv.js index aa6ddf26..806e7a74 100644 --- a/utils/worker-kv.js +++ b/utils/worker-kv.js @@ -1,4 +1,4 @@ -const {EventEmitter} = require('events'); +//const {EventEmitter} = require('events'); const zlib = require('zlib'); class WorkerKV { @@ -6,8 +6,8 @@ class WorkerKV { this.cache = false; this.loading = false; this.kvName = kvName; - this.events = new EventEmitter(); - this.events.setMaxListeners(0); + //this.events = new EventEmitter(); + //this.events.setMaxListeners(0); } async init(){ @@ -17,7 +17,7 @@ class WorkerKV { } if (this.loading) { //console.log('waiting for load', this.kvName); - return new Promise((resolve, reject) => { + /*return new Promise((resolve, reject) => { try { this.events.once('loaded', () => { //console.log(this.kvName, 'loaded event handled'); @@ -27,8 +27,8 @@ class WorkerKV { console.log(error); reject(error); } - }); - /*return new Promise((resolve) => { + });*/ + return new Promise((resolve) => { const isDone = () => { if (this.loading === false) { resolve(); @@ -42,7 +42,7 @@ class WorkerKV { } } isDone(); - });*/ + }); } this.loading = true; console.time('kv load '+this.kvName); @@ -63,7 +63,7 @@ class WorkerKV { this.loading = false; //console.log(this.kvName, 'listeners', this.events.listenerCount('loaded')); try { - this.events.emit('loaded'); + //this.events.emit('loaded'); } catch (error) { console.log(error); } From 9cd75dec1ff292df7433e6ee2251650127e3d202 Mon Sep 17 00:00:00 2001 From: Razzmatazz Date: Sat, 18 Jun 2022 23:00:28 -0500 Subject: [PATCH 05/22] fix sellFor currencyItem --- datasources/items.js | 1 + 1 file changed, 1 insertion(+) diff --git a/datasources/items.js b/datasources/items.js index ef717915..133971aa 100644 --- a/datasources/items.js +++ b/datasources/items.js @@ -16,6 +16,7 @@ class ItemsAPI extends WorkerKV { return { price: traderPrice.price, currency: traderPrice.currency, + currencyItem: traderPrice.currencyItem, priceRUB: traderPrice.priceRUB, vendor: { trader: traderPrice.trader, From 2f3736b209a21880a8153634ad95eb68f2348c85 Mon Sep 17 00:00:00 2001 From: Razzmatazz Date: Sun, 19 Jun 2022 06:52:19 -0500 Subject: [PATCH 06/22] try to circumvent cache --- utils/cache-machine.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/utils/cache-machine.js b/utils/cache-machine.js index a0fc2aa0..29834b62 100644 --- a/utils/cache-machine.js +++ b/utils/cache-machine.js @@ -1,8 +1,12 @@ // cache url const cacheUrl = 'https://cache.tarkov.dev' +let authToken = ''; +if (ENVIRONMENT === 'production') { + authToken = CACHE_BASIC_AUTH; +} const headers = { 'content-type': 'application/json;charset=UTF-8', - 'Authorization': `Basic ${CACHE_BASIC_AUTH}` + 'Authorization': `Basic ${authToken}` }; // Helper function to create a hash from a string From 466caa6b637cca34d41e2f5ff50b5799cef36dea Mon Sep 17 00:00:00 2001 From: Razzmatazz Date: Sun, 19 Jun 2022 06:58:00 -0500 Subject: [PATCH 07/22] comment vars --- wrangler.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wrangler.toml b/wrangler.toml index 750fa6a3..e4405858 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -20,7 +20,7 @@ kv-namespaces = [ { binding = "DATA_CACHE", id = "17fd725f04984e408d4a70b37c817171", preview_id = "17fd725f04984e408d4a70b37c817171" }, { binding = "QUERY_CACHE", id = "cf717460f7eb40299a0f8f844d157051", preview_id = "cf717460f7eb40299a0f8f844d157051" } ] -vars = { ENVIRONMENT = "staging" } +#vars = { ENVIRONMENT = "staging" } [env.development] kv-namespaces = [ From 69ad9ebb671544b13462d012bee2607260dca593 Mon Sep 17 00:00:00 2001 From: Razzmatazz Date: Sun, 19 Jun 2022 07:10:26 -0500 Subject: [PATCH 08/22] revert cache bypass --- utils/cache-machine.js | 6 +----- wrangler.toml | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/utils/cache-machine.js b/utils/cache-machine.js index 29834b62..a0fc2aa0 100644 --- a/utils/cache-machine.js +++ b/utils/cache-machine.js @@ -1,12 +1,8 @@ // cache url const cacheUrl = 'https://cache.tarkov.dev' -let authToken = ''; -if (ENVIRONMENT === 'production') { - authToken = CACHE_BASIC_AUTH; -} const headers = { 'content-type': 'application/json;charset=UTF-8', - 'Authorization': `Basic ${authToken}` + 'Authorization': `Basic ${CACHE_BASIC_AUTH}` }; // Helper function to create a hash from a string diff --git a/wrangler.toml b/wrangler.toml index e4405858..750fa6a3 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -20,7 +20,7 @@ kv-namespaces = [ { binding = "DATA_CACHE", id = "17fd725f04984e408d4a70b37c817171", preview_id = "17fd725f04984e408d4a70b37c817171" }, { binding = "QUERY_CACHE", id = "cf717460f7eb40299a0f8f844d157051", preview_id = "cf717460f7eb40299a0f8f844d157051" } ] -#vars = { ENVIRONMENT = "staging" } +vars = { ENVIRONMENT = "staging" } [env.development] kv-namespaces = [ From 7d3861e060715226e4e8ae30e5c47724c252cf16 Mon Sep 17 00:00:00 2001 From: Razzmatazz Date: Sun, 19 Jun 2022 07:54:28 -0500 Subject: [PATCH 09/22] add route --- wrangler.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wrangler.toml b/wrangler.toml index 750fa6a3..1128c9e2 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -15,7 +15,7 @@ kv-namespaces = [ vars = { ENVIRONMENT = "production" } [env.staging] -routes = ["dev.api.tarkov.dev/graphql", "dev.api.tarkov.dev/graphql*", "dev.api.tarkov.dev/___graphql", "dev.api.tarkov.dev/webhook*"] +routes = ["dev-api.tarkov.dev/*", "dev.api.tarkov.dev/graphql", "dev.api.tarkov.dev/graphql*", "dev.api.tarkov.dev/___graphql", "dev.api.tarkov.dev/webhook*"] kv-namespaces = [ { binding = "DATA_CACHE", id = "17fd725f04984e408d4a70b37c817171", preview_id = "17fd725f04984e408d4a70b37c817171" }, { binding = "QUERY_CACHE", id = "cf717460f7eb40299a0f8f844d157051", preview_id = "cf717460f7eb40299a0f8f844d157051" } From aa1611e51905e0193da730efe8b5a2712edbb6c9 Mon Sep 17 00:00:00 2001 From: Razzmatazz Date: Sun, 19 Jun 2022 07:57:25 -0500 Subject: [PATCH 10/22] comment routes --- wrangler.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wrangler.toml b/wrangler.toml index 1128c9e2..8fbfbab0 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -15,7 +15,7 @@ kv-namespaces = [ vars = { ENVIRONMENT = "production" } [env.staging] -routes = ["dev-api.tarkov.dev/*", "dev.api.tarkov.dev/graphql", "dev.api.tarkov.dev/graphql*", "dev.api.tarkov.dev/___graphql", "dev.api.tarkov.dev/webhook*"] +#routes = ["dev-api.tarkov.dev/*", "dev.api.tarkov.dev/graphql", "dev.api.tarkov.dev/graphql*", "dev.api.tarkov.dev/___graphql", "dev.api.tarkov.dev/webhook*"] kv-namespaces = [ { binding = "DATA_CACHE", id = "17fd725f04984e408d4a70b37c817171", preview_id = "17fd725f04984e408d4a70b37c817171" }, { binding = "QUERY_CACHE", id = "cf717460f7eb40299a0f8f844d157051", preview_id = "cf717460f7eb40299a0f8f844d157051" } From f36bfa2f5bb439151c6a4dcfd6f1d2111f846516 Mon Sep 17 00:00:00 2001 From: Razzmatazz Date: Sun, 19 Jun 2022 08:07:26 -0500 Subject: [PATCH 11/22] uncomment routes --- wrangler.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wrangler.toml b/wrangler.toml index 8fbfbab0..1128c9e2 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -15,7 +15,7 @@ kv-namespaces = [ vars = { ENVIRONMENT = "production" } [env.staging] -#routes = ["dev-api.tarkov.dev/*", "dev.api.tarkov.dev/graphql", "dev.api.tarkov.dev/graphql*", "dev.api.tarkov.dev/___graphql", "dev.api.tarkov.dev/webhook*"] +routes = ["dev-api.tarkov.dev/*", "dev.api.tarkov.dev/graphql", "dev.api.tarkov.dev/graphql*", "dev.api.tarkov.dev/___graphql", "dev.api.tarkov.dev/webhook*"] kv-namespaces = [ { binding = "DATA_CACHE", id = "17fd725f04984e408d4a70b37c817171", preview_id = "17fd725f04984e408d4a70b37c817171" }, { binding = "QUERY_CACHE", id = "cf717460f7eb40299a0f8f844d157051", preview_id = "cf717460f7eb40299a0f8f844d157051" } From bbd682428f97286fe6265ed5bd5571e50eb226d6 Mon Sep 17 00:00:00 2001 From: Razzmatazz Date: Sun, 19 Jun 2022 08:18:19 -0500 Subject: [PATCH 12/22] force deployh From 3a160585a82de322275634804af62a3133ad1338 Mon Sep 17 00:00:00 2001 From: Razzmatazz Date: Sun, 19 Jun 2022 08:43:56 -0500 Subject: [PATCH 13/22] try this --- .github/workflows/pull_request.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 797f2a79..1400cc77 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -6,7 +6,7 @@ on: jobs: deploy-developement: - environment: development + environment: staging runs-on: ubuntu-latest steps: From 7fbf13df8f7fe458c10c13a16ecd13bf1027721a Mon Sep 17 00:00:00 2001 From: Razzmatazz Date: Mon, 20 Jun 2022 07:17:48 -0500 Subject: [PATCH 14/22] fix dev deploy --- .github/workflows/pull_request.yml | 6 +++--- package.json | 4 ++-- script/ci/wrangler-ci.toml | 2 +- webpack.config.js | 2 +- webpack.dev.js | 19 +++++++++++++++++++ wrangler.toml | 15 +++++++++------ 6 files changed, 35 insertions(+), 13 deletions(-) create mode 100644 webpack.dev.js diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 1400cc77..5c93ddfb 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -5,8 +5,8 @@ on: branches: [main] jobs: - deploy-developement: - environment: staging + deploy-development: + environment: development runs-on: ubuntu-latest steps: @@ -30,7 +30,7 @@ jobs: uses: cloudflare/wrangler-action@3424d15af26edad39d5276be3cc0cc9ffec22b55 # pin@1.3.0 with: apiToken: ${{ secrets.CF_API_TOKEN }} - environment: "staging" + environment: "development" # Post comment on PR with development deploy info - uses: actions/github-script@5d03ada4b0a753e9460b312e61cc4f8fdeacf163 # pin@0.9.0 diff --git a/package.json b/package.json index 46e3df3a..8e2ddbd4 100644 --- a/package.json +++ b/package.json @@ -6,8 +6,8 @@ "scripts": { "test": "echo \"Error: no test specified\"", "format": "prettier --write '**/*.{js,css,json,md}'", - "dev": "wrangler --env development dev", - "ci": "wrangler --env development dev", + "dev": "wrangler --env development_local dev", + "ci": "wrangler --env development_local dev", "local": "wrangler dev" }, "author": "Oskar Risberg ", diff --git a/script/ci/wrangler-ci.toml b/script/ci/wrangler-ci.toml index 98be34e7..cfa4d5e7 100644 --- a/script/ci/wrangler-ci.toml +++ b/script/ci/wrangler-ci.toml @@ -11,7 +11,7 @@ kv-namespaces = [ { binding = "QUERY_CACHE", id = "7ae67967cff24ceaa31ca58c5a6da798", preview_id = "7ae67967cff24ceaa31ca58c5a6da798" } ] -[env.development] +[env.development_local] # routes = ["dev.api.tarkov.dev/graphql", "dev.api.tarkov.dev/graphql*", "dev.api.tarkov.dev/___graphql", "dev.api.tarkov.dev/webhook*"] kv-namespaces = [ { binding = "DATA_CACHE", id = "17fd725f04984e408d4a70b37c817171", preview_id = "17fd725f04984e408d4a70b37c817171" }, diff --git a/webpack.config.js b/webpack.config.js index f8407520..7b220c1b 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -1,5 +1,5 @@ const config = { - mode: 'none', // "production" | "development" | "none" + mode: 'production', // "production" | "development" | "none" resolve: { extensions: ['*', '.mjs', '.js', '.json'] }, diff --git a/webpack.dev.js b/webpack.dev.js new file mode 100644 index 00000000..f8407520 --- /dev/null +++ b/webpack.dev.js @@ -0,0 +1,19 @@ +const config = { + mode: 'none', // "production" | "development" | "none" + resolve: { + extensions: ['*', '.mjs', '.js', '.json'] + }, + target: 'webworker', + entry: './index.js', + module: { + rules: [ + { + test: /\.mjs$/, + include: /node_modules/, + type: 'javascript/auto' + } + ] + } +}; + +module.exports = config; diff --git a/wrangler.toml b/wrangler.toml index 1128c9e2..6ccc532b 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -4,9 +4,10 @@ type = "webpack" account_id = "424ad63426a1ae47d559873f929eb9fc" zone_id = "777d475095752ccbe5d1d49e690a305b" workers_dev = true -webpack_config = "webpack.config.js" +webpack_config = "webpack.dev.js" [env.production] +webpack_config = "webpack.config.js" routes = ["api.tarkov.dev/graphql", "api.tarkov.dev/graphql*", "api.tarkov.dev/___graphql", "api.tarkov.dev/webhook*"] kv-namespaces = [ { binding = "DATA_CACHE", id = "2e6feba88a9e4097b6d2209191ed4ae5", preview_id = "17fd725f04984e408d4a70b37c817171" }, @@ -14,20 +15,22 @@ kv-namespaces = [ ] vars = { ENVIRONMENT = "production" } -[env.staging] -routes = ["dev-api.tarkov.dev/*", "dev.api.tarkov.dev/graphql", "dev.api.tarkov.dev/graphql*", "dev.api.tarkov.dev/___graphql", "dev.api.tarkov.dev/webhook*"] +[env.development] +#routes = ["dev.api.tarkov.dev/graphql", "dev.api.tarkov.dev/graphql*", "dev.api.tarkov.dev/___graphql", "dev.api.tarkov.dev/webhook*"] kv-namespaces = [ { binding = "DATA_CACHE", id = "17fd725f04984e408d4a70b37c817171", preview_id = "17fd725f04984e408d4a70b37c817171" }, { binding = "QUERY_CACHE", id = "cf717460f7eb40299a0f8f844d157051", preview_id = "cf717460f7eb40299a0f8f844d157051" } ] -vars = { ENVIRONMENT = "staging" } +vars = { ENVIRONMENT = "development" } -[env.development] +[env.development_local] kv-namespaces = [ { binding = "DATA_CACHE", id = "2e6feba88a9e4097b6d2209191ed4ae5", preview_id = "17fd725f04984e408d4a70b37c817171" }, { binding = "QUERY_CACHE", id = "7ae67967cff24ceaa31ca58c5a6da798", preview_id = "cf717460f7eb40299a0f8f844d157051" } ] -vars = { ENVIRONMENT = "development" } +vars = { ENVIRONMENT = "development_local" } # [secrets] # CACHE_BASIC_AUTH +# TWITCH_CLIENT_ID +# TWITCH_TOKEN From 8baa68e50f932aacc5c4e6908fe13edf60c9b688 Mon Sep 17 00:00:00 2001 From: Razzmatazz Date: Mon, 20 Jun 2022 07:41:16 -0500 Subject: [PATCH 15/22] fix deploy pls --- package.json | 4 ++-- script/ci/wrangler-ci.toml | 2 +- wrangler.toml | 7 ------- 3 files changed, 3 insertions(+), 10 deletions(-) diff --git a/package.json b/package.json index 8e2ddbd4..46e3df3a 100644 --- a/package.json +++ b/package.json @@ -6,8 +6,8 @@ "scripts": { "test": "echo \"Error: no test specified\"", "format": "prettier --write '**/*.{js,css,json,md}'", - "dev": "wrangler --env development_local dev", - "ci": "wrangler --env development_local dev", + "dev": "wrangler --env development dev", + "ci": "wrangler --env development dev", "local": "wrangler dev" }, "author": "Oskar Risberg ", diff --git a/script/ci/wrangler-ci.toml b/script/ci/wrangler-ci.toml index cfa4d5e7..98be34e7 100644 --- a/script/ci/wrangler-ci.toml +++ b/script/ci/wrangler-ci.toml @@ -11,7 +11,7 @@ kv-namespaces = [ { binding = "QUERY_CACHE", id = "7ae67967cff24ceaa31ca58c5a6da798", preview_id = "7ae67967cff24ceaa31ca58c5a6da798" } ] -[env.development_local] +[env.development] # routes = ["dev.api.tarkov.dev/graphql", "dev.api.tarkov.dev/graphql*", "dev.api.tarkov.dev/___graphql", "dev.api.tarkov.dev/webhook*"] kv-namespaces = [ { binding = "DATA_CACHE", id = "17fd725f04984e408d4a70b37c817171", preview_id = "17fd725f04984e408d4a70b37c817171" }, diff --git a/wrangler.toml b/wrangler.toml index 6ccc532b..b8d956fa 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -23,13 +23,6 @@ kv-namespaces = [ ] vars = { ENVIRONMENT = "development" } -[env.development_local] -kv-namespaces = [ - { binding = "DATA_CACHE", id = "2e6feba88a9e4097b6d2209191ed4ae5", preview_id = "17fd725f04984e408d4a70b37c817171" }, - { binding = "QUERY_CACHE", id = "7ae67967cff24ceaa31ca58c5a6da798", preview_id = "cf717460f7eb40299a0f8f844d157051" } -] -vars = { ENVIRONMENT = "development_local" } - # [secrets] # CACHE_BASIC_AUTH # TWITCH_CLIENT_ID From 041d989beac3e32d4e6defa88391375580ea33a7 Mon Sep 17 00:00:00 2001 From: Razzmatazz Date: Mon, 20 Jun 2022 07:49:33 -0500 Subject: [PATCH 16/22] use live data for ci test --- datasources/hideout-legacy.js | 62 ----------------------------------- datasources/index.js | 5 --- script/ci/wrangler-ci.toml | 4 +-- 3 files changed, 2 insertions(+), 69 deletions(-) delete mode 100644 datasources/hideout-legacy.js diff --git a/datasources/hideout-legacy.js b/datasources/hideout-legacy.js deleted file mode 100644 index f5b7b5a4..00000000 --- a/datasources/hideout-legacy.js +++ /dev/null @@ -1,62 +0,0 @@ -class HideoutLegacyAPI { - constructor(){ - this.moduleList = false; - } - - async getList(){ - if(this.moduleList){ - return this.moduleList; - } - - const hideoutData = await DATA_CACHE.get('hideout_legacy_data', 'json'); - const returnData = []; - - for(const hideoutModule of hideoutData.data){ - const newRequirement = { - id: hideoutModule.id, - name: hideoutModule.module, - level: hideoutModule.level, - itemRequirements: hideoutModule.require.map((hideoutRequirement) => { - if(hideoutRequirement.type !== 'item'){ - return false; - } - - return { - item: hideoutRequirement.name, - quantity: hideoutRequirement.quantity, - count: hideoutRequirement.quantity, - }; - }), - moduleRequirements: hideoutModule.require.map((hideoutRequirement) => { - if(hideoutRequirement.type !== 'module'){ - return false; - } - - return { - name: hideoutRequirement.name, - level: hideoutRequirement.quantity, - }; - }).filter(Boolean), - }; - - newRequirement.itemRequirements = newRequirement.itemRequirements.filter(Boolean); - returnData.push(newRequirement); - } - - for(const hideoutModule of returnData){ - hideoutModule.moduleRequirements = hideoutModule.moduleRequirements.map((basicModuleObject) => { - return this.getModule(basicModuleObject.name, basicModuleObject.level, returnData); - }); - } - - this.moduleList = returnData; - - return returnData; - } - - async getModule(name, level, moduleList) { - return moduleList.find(hideoutModule => hideoutModule.name === name && hideoutModule.level === level); - } -} - -module.exports = HideoutLegacyAPI; diff --git a/datasources/index.js b/datasources/index.js index a481b149..c2f980d8 100644 --- a/datasources/index.js +++ b/datasources/index.js @@ -1,7 +1,5 @@ -//const AmmoAPI = require('./ammo'); const BartersAPI = require('./barters'); const CraftsAPI = require('./crafts'); -//const HideoutLegacyAPI = require('./hideout-legacy'); const HideoutAPI = require('./hideout'); const HistoricalPricesAPI = require('./historical-prices'); const ItemsAPI = require('./items'); @@ -14,15 +12,12 @@ const TradersAPI = require('./traders'); class DataSource { constructor(){ - //this.ammo = new AmmoAPI(); this.barter = new BartersAPI(); this.craft = new CraftsAPI(); - //this.hideoutLegacy = new HideoutLegacyAPI(); this.hideout = new HideoutAPI(); this.historicalPrice = new HistoricalPricesAPI(); this.item = new ItemsAPI(); this.map = new MapAPI(); - //this.quest = new QuestsAPI(); this.status = status; this.traderInventory = new TraderInventoryAPI(); this.trader = new TradersAPI(); diff --git a/script/ci/wrangler-ci.toml b/script/ci/wrangler-ci.toml index 98be34e7..981d1344 100644 --- a/script/ci/wrangler-ci.toml +++ b/script/ci/wrangler-ci.toml @@ -14,8 +14,8 @@ kv-namespaces = [ [env.development] # routes = ["dev.api.tarkov.dev/graphql", "dev.api.tarkov.dev/graphql*", "dev.api.tarkov.dev/___graphql", "dev.api.tarkov.dev/webhook*"] kv-namespaces = [ - { binding = "DATA_CACHE", id = "17fd725f04984e408d4a70b37c817171", preview_id = "17fd725f04984e408d4a70b37c817171" }, - { binding = "QUERY_CACHE", id = "cf717460f7eb40299a0f8f844d157051", preview_id = "cf717460f7eb40299a0f8f844d157051" } + { binding = "DATA_CACHE", id = "2e6feba88a9e4097b6d2209191ed4ae5", preview_id = "2e6feba88a9e4097b6d2209191ed4ae5" }, + { binding = "QUERY_CACHE", id = "7ae67967cff24ceaa31ca58c5a6da798", preview_id = "7ae67967cff24ceaa31ca58c5a6da798" } ] # [secrets] From 41811fdeab8a1add36107163de335562c55c1af5 Mon Sep 17 00:00:00 2001 From: Razzmatazz Date: Mon, 20 Jun 2022 07:51:16 -0500 Subject: [PATCH 17/22] comment debug output --- index.js | 14 +++++++------- utils/worker-kv.js | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/index.js b/index.js index 668ae69b..fc75a76d 100644 --- a/index.js +++ b/index.js @@ -64,8 +64,8 @@ async function getSchema(data) { } addEventListener('fetch', event => { - console.time(event.request.cf.tlsExportedAuthenticator.clientFinished+' total response'); - console.log('environment', ENVIRONMENT); + //console.time(event.request.cf.tlsExportedAuthenticator.clientFinished+' total response'); + //console.log('environment', ENVIRONMENT); event.respondWith(handleRequest(event)); }); @@ -79,9 +79,9 @@ async function graphqlHandler(event, graphQLOptions) { if (request.method === 'POST') { try { //console.timeEnd(event.request.cf.tlsExportedAuthenticator.clientFinished+' checkPoint'); - console.time(event.request.cf.tlsExportedAuthenticator.clientFinished+' request.json'); + //console.time(event.request.cf.tlsExportedAuthenticator.clientFinished+' request.json'); const requestBody = await request.json(); - console.timeEnd(event.request.cf.tlsExportedAuthenticator.clientFinished+' request.json'); + //console.timeEnd(event.request.cf.tlsExportedAuthenticator.clientFinished+' request.json'); query = requestBody.query; variables = requestBody.variables; } catch (jsonError) { @@ -150,13 +150,13 @@ async function graphqlHandler(event, graphQLOptions) { } } */ - console.time(event.request.cf.tlsExportedAuthenticator.clientFinished+' init') + //console.time(event.request.cf.tlsExportedAuthenticator.clientFinished+' init') try { await dataAPI.init(); } catch (error) { console.log('init error', error, error.stack); } - console.timeEnd(event.request.cf.tlsExportedAuthenticator.clientFinished+' init') + //console.timeEnd(event.request.cf.tlsExportedAuthenticator.clientFinished+' init') const result = await graphql(await getSchema(dataAPI), query, {}, {data: dataAPI, util: graphqlUtil}, variables); const body = JSON.stringify(result); @@ -169,7 +169,7 @@ async function graphqlHandler(event, graphQLOptions) { /* if(!result.errors && !url.hostname.includes('localhost') && !url.hostname.includes('tutorial.cloudflareworkers.com')){ await QUERY_CACHE.put(queryHash, body, {expirationTtl: 300}); } */ - console.timeEnd(event.request.cf.tlsExportedAuthenticator.clientFinished+' total response') + //console.timeEnd(event.request.cf.tlsExportedAuthenticator.clientFinished+' total response') return new Response(body, { headers: { 'content-type': 'application/json', diff --git a/utils/worker-kv.js b/utils/worker-kv.js index 806e7a74..54065d87 100644 --- a/utils/worker-kv.js +++ b/utils/worker-kv.js @@ -45,12 +45,12 @@ class WorkerKV { }); } this.loading = true; - console.time('kv load '+this.kvName); + //console.time('kv load '+this.kvName); return new Promise((resolve, reject) => { DATA_CACHE.getWithMetadata(this.kvName, 'text').then(response => { const data = response.value; const metadata = response.metadata; - console.timeEnd('kv load '+this.kvName); + //console.timeEnd('kv load '+this.kvName); if (metadata && metadata.compression) { if (metadata.compression = 'gzip') { this.cache = JSON.parse(zlib.gunzipSync(Buffer.from(data, metadata.encoding)).toString()); From 99efa324336bd55e2dffaab75ad376ccbb02b5b4 Mon Sep 17 00:00:00 2001 From: Razzmatazz Date: Mon, 20 Jun 2022 08:02:12 -0500 Subject: [PATCH 18/22] un-fix typo --- .github/workflows/pull_request.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 5c93ddfb..37894510 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -5,7 +5,7 @@ on: branches: [main] jobs: - deploy-development: + deploy-developement: environment: development runs-on: ubuntu-latest From 73d56eb003c39ba0244574faaae539a1f35e7f5c Mon Sep 17 00:00:00 2001 From: Razzmatazz Date: Mon, 20 Jun 2022 09:01:14 -0500 Subject: [PATCH 19/22] don't need separate ci toml --- script/ci/wrangler-ci.toml | 22 ---------------------- 1 file changed, 22 deletions(-) delete mode 100644 script/ci/wrangler-ci.toml diff --git a/script/ci/wrangler-ci.toml b/script/ci/wrangler-ci.toml deleted file mode 100644 index 981d1344..00000000 --- a/script/ci/wrangler-ci.toml +++ /dev/null @@ -1,22 +0,0 @@ -compatibility_date = "2021-12-14" -name = "api" -type = "webpack" -account_id = "424ad63426a1ae47d559873f929eb9fc" -zone_id = "777d475095752ccbe5d1d49e690a305b" -workers_dev = true -webpack_config = "webpack.config.js" -# routes = ["api.tarkov.dev/graphql", "api.tarkov.dev/graphql*", "api.tarkov.dev/___graphql", "api.tarkov.dev/webhook*"] -kv-namespaces = [ - { binding = "DATA_CACHE", id = "2e6feba88a9e4097b6d2209191ed4ae5", preview_id = "2e6feba88a9e4097b6d2209191ed4ae5" }, - { binding = "QUERY_CACHE", id = "7ae67967cff24ceaa31ca58c5a6da798", preview_id = "7ae67967cff24ceaa31ca58c5a6da798" } -] - -[env.development] -# routes = ["dev.api.tarkov.dev/graphql", "dev.api.tarkov.dev/graphql*", "dev.api.tarkov.dev/___graphql", "dev.api.tarkov.dev/webhook*"] -kv-namespaces = [ - { binding = "DATA_CACHE", id = "2e6feba88a9e4097b6d2209191ed4ae5", preview_id = "2e6feba88a9e4097b6d2209191ed4ae5" }, - { binding = "QUERY_CACHE", id = "7ae67967cff24ceaa31ca58c5a6da798", preview_id = "7ae67967cff24ceaa31ca58c5a6da798" } -] - -# [secrets] -# CACHE_BASIC_AUTH From 340cf42162d8888b2b9962edb40c9410dcdf7f13 Mon Sep 17 00:00:00 2001 From: Razzmatazz Date: Mon, 20 Jun 2022 09:13:05 -0500 Subject: [PATCH 20/22] dynamic load kvs --- index.js | 2 +- utils/worker-kv.js | 15 +++++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index fc75a76d..ac5b6375 100644 --- a/index.js +++ b/index.js @@ -152,7 +152,7 @@ async function graphqlHandler(event, graphQLOptions) { //console.time(event.request.cf.tlsExportedAuthenticator.clientFinished+' init') try { - await dataAPI.init(); + //await dataAPI.init(); } catch (error) { console.log('init error', error, error.stack); } diff --git a/utils/worker-kv.js b/utils/worker-kv.js index 54065d87..14a14478 100644 --- a/utils/worker-kv.js +++ b/utils/worker-kv.js @@ -6,6 +6,8 @@ class WorkerKV { this.cache = false; this.loading = false; this.kvName = kvName; + this.loadingPromises = []; + this.loadingInterval = false; //this.events = new EventEmitter(); //this.events.setMaxListeners(0); } @@ -29,7 +31,7 @@ class WorkerKV { } });*/ return new Promise((resolve) => { - const isDone = () => { + /*const isDone = () => { if (this.loading === false) { resolve(); } else { @@ -41,10 +43,19 @@ class WorkerKV { } } } - isDone(); + isDone();*/ + this.loadingPromises.push(resolve); }); } this.loading = true; + this.loadingInterval = setInterval(() => { + if (this.loading) return; + let resolve = false; + while (resolve = this.loadingPromises.shift()) { + resolve(); + } + clearInterval(this.loadingInterval); + }, 5); //console.time('kv load '+this.kvName); return new Promise((resolve, reject) => { DATA_CACHE.getWithMetadata(this.kvName, 'text').then(response => { From 046ee6b59648d30340fe9e2e9854c499b79554f0 Mon Sep 17 00:00:00 2001 From: Razzmatazz Date: Mon, 20 Jun 2022 09:30:41 -0500 Subject: [PATCH 21/22] clean up --- index.js | 15 +++------------ utils/worker-kv.js | 33 --------------------------------- 2 files changed, 3 insertions(+), 45 deletions(-) diff --git a/index.js b/index.js index ac5b6375..7a282fd8 100644 --- a/index.js +++ b/index.js @@ -1,5 +1,4 @@ //const crypto = require('crypto'); -const { EventEmitter } = require('events'); const { makeExecutableSchema } = require('@graphql-tools/schema'); const { mergeTypeDefs } = require('@graphql-tools/merge'); @@ -23,8 +22,6 @@ const twitch = require('./custom-endpoints/twitch'); let schema = false; let loadingSchema = false; const skipCache = ENVIRONMENT !== 'production' || false; -//const schemaEvents = new EventEmitter(); -//schemaEvents.setMaxListeners(0); /** * Example of how router can be used in an application @@ -35,11 +32,6 @@ async function getSchema(data) { return schema; } if (loadingSchema) { - /*return new Promise((resolve) => { - schemaEvents.once('loaded', () => { - resolve(schema); - }); - });*/ return new Promise((resolve) => { const isDone = () => { if (this.loadingSchema === false) { @@ -55,7 +47,6 @@ async function getSchema(data) { return dynamicTypeDefs(data).then(dynamicTypeDefs => { schema = makeExecutableSchema({typeDefs: mergeTypeDefs([typeDefs, dynamicTypeDefs]), resolvers: resolvers}); loadingSchema = false; - //schemaEvents.emit('loaded'); return schema; }).catch(error => { loadingSchema = false; @@ -151,11 +142,11 @@ async function graphqlHandler(event, graphQLOptions) { } */ //console.time(event.request.cf.tlsExportedAuthenticator.clientFinished+' init') - try { - //await dataAPI.init(); + /*try { + await dataAPI.init(); } catch (error) { console.log('init error', error, error.stack); - } + }*/ //console.timeEnd(event.request.cf.tlsExportedAuthenticator.clientFinished+' init') const result = await graphql(await getSchema(dataAPI), query, {}, {data: dataAPI, util: graphqlUtil}, variables); const body = JSON.stringify(result); diff --git a/utils/worker-kv.js b/utils/worker-kv.js index 14a14478..3b9b42f9 100644 --- a/utils/worker-kv.js +++ b/utils/worker-kv.js @@ -1,4 +1,3 @@ -//const {EventEmitter} = require('events'); const zlib = require('zlib'); class WorkerKV { @@ -8,8 +7,6 @@ class WorkerKV { this.kvName = kvName; this.loadingPromises = []; this.loadingInterval = false; - //this.events = new EventEmitter(); - //this.events.setMaxListeners(0); } async init(){ @@ -18,32 +15,7 @@ class WorkerKV { return; } if (this.loading) { - //console.log('waiting for load', this.kvName); - /*return new Promise((resolve, reject) => { - try { - this.events.once('loaded', () => { - //console.log(this.kvName, 'loaded event handled'); - resolve(); - }); - } catch (error) { - console.log(error); - reject(error); - } - });*/ return new Promise((resolve) => { - /*const isDone = () => { - if (this.loading === false) { - resolve(); - } else { - try { - setTimeout(isDone, 5); - } catch (error) { - console.log(error.stack); - reject(error); - } - } - } - isDone();*/ this.loadingPromises.push(resolve); }); } @@ -73,11 +45,6 @@ class WorkerKV { } this.loading = false; //console.log(this.kvName, 'listeners', this.events.listenerCount('loaded')); - try { - //this.events.emit('loaded'); - } catch (error) { - console.log(error); - } resolve(); }).catch(error => { this.loading = false; From 7f04cda8150408a39e3be684598da6022c627eb4 Mon Sep 17 00:00:00 2001 From: Razzmatazz Date: Mon, 20 Jun 2022 16:49:52 -0500 Subject: [PATCH 22/22] fix production --- wrangler.toml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/wrangler.toml b/wrangler.toml index b8d956fa..4573fdc6 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -4,9 +4,6 @@ type = "webpack" account_id = "424ad63426a1ae47d559873f929eb9fc" zone_id = "777d475095752ccbe5d1d49e690a305b" workers_dev = true -webpack_config = "webpack.dev.js" - -[env.production] webpack_config = "webpack.config.js" routes = ["api.tarkov.dev/graphql", "api.tarkov.dev/graphql*", "api.tarkov.dev/___graphql", "api.tarkov.dev/webhook*"] kv-namespaces = [ @@ -15,8 +12,10 @@ kv-namespaces = [ ] vars = { ENVIRONMENT = "production" } + [env.development] -#routes = ["dev.api.tarkov.dev/graphql", "dev.api.tarkov.dev/graphql*", "dev.api.tarkov.dev/___graphql", "dev.api.tarkov.dev/webhook*"] +webpack_config = "webpack.dev.js" +routes = [] kv-namespaces = [ { binding = "DATA_CACHE", id = "17fd725f04984e408d4a70b37c817171", preview_id = "17fd725f04984e408d4a70b37c817171" }, { binding = "QUERY_CACHE", id = "cf717460f7eb40299a0f8f844d157051", preview_id = "cf717460f7eb40299a0f8f844d157051" }