From fed9e02133d02baf0ebeb85c574d462a6ddf5ed0 Mon Sep 17 00:00:00 2001 From: Kanaxai Date: Mon, 7 Sep 2026 03:28:48 -0300 Subject: [PATCH 1/4] Fix Price item searches from trade URLs --- src/Classes/TradeQueryRequests.lua | 1147 ++++++++++++++-------------- 1 file changed, 585 insertions(+), 562 deletions(-) diff --git a/src/Classes/TradeQueryRequests.lua b/src/Classes/TradeQueryRequests.lua index 56e3720b71..7cbeb47833 100644 --- a/src/Classes/TradeQueryRequests.lua +++ b/src/Classes/TradeQueryRequests.lua @@ -1,562 +1,585 @@ --- Path of Building --- --- Module: Trade Query Requests --- Handling trade api requests while respecting rate limits --- - -local dkjson = require "dkjson" -local utils = LoadModule("Modules/Utils") - ----@class TradeQueryRequests ----@class TradeQueryRequests -local TradeQueryRequestsClass = newClass("TradeQueryRequests") - -function TradeQueryRequestsClass:TradeQueryRequests(rateLimiter) - self.maxFetchPerSearch = 10 - self.tradeQuery = tradeQuery - self.rateLimiter = rateLimiter or new("TradeQueryRateLimiter"):TradeQueryRateLimiter() - self.requestQueue = { - ["search"] = {}, - ["fetch"] = {}, - } - self.hostName = "https://www.pathofexile.com/" - return self -end - ----Main routine for processing request queue ---- @param onRateLimit fun(integer)? -function TradeQueryRequestsClass:ProcessQueue(onRateLimit) - for key, queue in pairs(self.requestQueue) do - if #queue > 0 then - local policy = self.rateLimiter:GetPolicyName(key) - local now = os.time() - local timeNext = self.rateLimiter:NextRequestTime(policy, now) - local timeLeft = timeNext - now - -- relay wait info to caller when actually waiting, and not just - -- getting a magic poe2 release date number - if onRateLimit and timeLeft > 1 and timeNext ~= 1956528000 then - onRateLimit(timeLeft) - end - if not (queue[1].retryTime and now < queue[1].retryTime) then - if now >= timeNext then - local request = table.remove(queue, 1) - local requestId = self.rateLimiter:InsertRequest(policy) - local onComplete = function(response, errMsg) - self.rateLimiter:FinishRequest(policy, requestId) - self.rateLimiter:UpdateFromHeader(response.header, policy) - if response.header:match("HTTP/[%d%.]+ (%d+)") == "429" then - local retryAfter = response.header:match("Retry%-After:%s+(%d+)") - retryAfter = retryAfter and tonumber(retryAfter) or 0 - request.attempts = (request.attempts or 0) + 1 - - local backoff = math.max(math.min(2 ^ request.attempts, 60), retryAfter) - request.retryTime = os.time() + backoff - table.insert(queue, 1, request) - -- optional callback with the backoff time when rate - -- limited to inform user - if onRateLimit then - onRateLimit(backoff) - end - return - end - if errMsg == "Response code: 401" and response.body:find("invalid_token") then - errMsg = errMsg .. "\nAuthorization is invalid. Please Re-Log and reset" - main.api:ResetDetails() - end - request.callback(response.body, errMsg, unpack(request.callbackParams or {})) - end - local header = "Content-Type: application/json" - if main.api.authToken then - header = header .."\nAuthorization: Bearer "..main.api.authToken - end - launch:DownloadPage(request.url, onComplete, { - header = header, - body = request.body, - }) - else - break - end - end - end - end -end - ----Performs search and fetches results ----@param league string ----@param query string ----@param callback fun(items:table, errMsg:string) ----@param params table @ params = { callbackQueryId = fun(queryId:string) } -function TradeQueryRequestsClass:SearchWithQuery(realm, league, query, callback, params) - params = params or {} - --ConPrintf("Query json: %s", query) - self:PerformSearch(realm, league, query, function(response, errMsg) - if params.callbackQueryId and response and response.id then - params.callbackQueryId(response.id) - end - if errMsg then - return callback(nil, errMsg) - end - self:FetchResults(response.result, response.id, callback) - end) -end - ----Performs search and fetches results, adjusting the query weight and repeating ----the search to fetch more items when the search cap (10k items) is reached ----@param league string ----@param query string ----@param callback fun(items:table, errMsg:string) ----@param params table @ params = { callbackQueryId = fun(queryId:string) } -function TradeQueryRequestsClass:SearchWithQueryWeightAdjusted(realm, league, query, callback, params) - params = params or {} - local previousSearchId = nil - local previousSearchItemIds = nil - local previousSearchItems = nil - -- Limit recursion to prevent potential loops - -- Each repeat is a leap of 10k items, normally we shouldn't need more than 1-2 steps anyways - local maxRecursion = 5 - local currentRecursion = 0 - local function performSearchCallback(response, errMsg) - currentRecursion = currentRecursion + 1 - if params.callbackQueryId and response and response.id then - params.callbackQueryId(response.id) - end - if errMsg and ((errMsg == "No Matching Results Found" and currentRecursion >= maxRecursion) or errMsg ~= "No Matching Results Found") then - return callback(nil, errMsg) - end - if (response.total > self.maxFetchPerSearch and response.total < 10000) or currentRecursion >= maxRecursion then - -- Search not clipped or max recursion reached, fetch results and finalize - if previousSearchItems and self.maxFetchPerSearch > response.total then - -- Not enough items in the last search, fill results from previous search - self:FetchResults(response.result, response.id, function(items, errMsg) - if errMsg then - return callback(nil, errMsg) - end - local fetchedItemIds = {} - local idSet = {} - for _, value in pairs(items) do - if not idSet[value.id] then - idSet[value.id] = true - table.insert(fetchedItemIds, value.id) - end - end - for _, value in pairs(previousSearchItems) do - if #items >= self.maxFetchPerSearch then - break - end - if not isValueInTable(fetchedItemIds, value.id) then - table.insert(items, value) - table.insert(fetchedItemIds, value.id) - end - end - local fillCount = self.maxFetchPerSearch - #items - if fillCount > 0 then - -- fill with previous search results - local unfetchedItemIds = {} - for _, value in pairs(previousSearchItemIds) do - if #unfetchedItemIds >= fillCount then - break - end - if not isValueInTable(fetchedItemIds, value) then - table.insert(unfetchedItemIds, value) - end - end - self:FetchResults(unfetchedItemIds, previousSearchId, function(newItems, errMsg) - if errMsg then - return callback(nil, errMsg) - end - items = tableConcat(items, newItems) - callback(items, errMsg) - end) - else - callback(items, errMsg) - end - end) - else - -- Search not clipped and result count satisfy maxFetchPerSearch, proceed normally - self:FetchResults(response.result, response.id, callback) - end - else - if response.total < self.maxFetchPerSearch then -- Less than maximum items retrieved lower weight to try and get more. - local queryJson = dkjson.decode(query) - queryJson.query.stats[1].value.min = queryJson.query.stats[1].value.min / 2 - query = dkjson.encode(queryJson) - self:PerformSearch(realm, league, query, performSearchCallback) - else -- Search clipped, fetch highest weight item, update query weight and repeat search - previousSearchItemIds = response.result - previousSearchId = response.id - local firstResultBatch = {unpack(response.result, 1, math.min(#response.result, 10))} - self:FetchResults(firstResultBatch, response.id, function(items, errMsg) - if errMsg then - return callback(nil, errMsg) - end - previousSearchItems = items - local highestWeight = items[1].weight - local queryJson = dkjson.decode(query) - queryJson.query.stats[1].value.min = (tonumber(highestWeight) + queryJson.query.stats[1].value.min) / 2 - query = dkjson.encode(queryJson) - self:PerformSearch(realm, league, query, performSearchCallback) - end) - end - end - end - self:PerformSearch(realm, league, query, performSearchCallback) -end - ----Perform search and run callback function on returned item hashes. ----Item info has to be fetched separately ----@param league string ----@param query string ----@param callback fun(response:table, errMsg:string) -function TradeQueryRequestsClass:PerformSearch(realm, league, query, callback) - table.insert(self.requestQueue["search"], { - url = self:buildUrl(self.hostName .. "api/trade2/search", realm, league), - body = query, - callback = function(response, errMsg) - if errMsg and not errMsg:find("Response code: 400") then - return callback(nil, errMsg) - end - local response = dkjson.decode(response) - if not response then - errMsg = "Failed to Get Trade response" - return callback(nil, errMsg) - end - if not response.result or #response.result == 0 then - if response.error then - if not (response.error.code and response.error.message) then - errMsg = "Encountered unknown error, check console for details." - ConPrintf("Unknown error: %s", utils.stringify(response.error)) - callback(response, errMsg) - end - if response.error.message:find("Logging in will increase this limit") then - errMsg = "Authorization is invalid. Please Re-Log and reset" - else - -- Report unhandled error - errMsg = "[ " .. response.error.code .. ": " .. response.error.message .. " ]" - end - else - ConPrintf("Found 0 results for %sapi/trade2/search/%s/%s", self.hostName, league, response.id) - errMsg = "No Matching Results Found" - end - return callback(response, errMsg) - end - callback(response, errMsg) - end, - }) -end - ----Fetch item details for itemHashes ----@param itemHashes string[] ----@param queryId string ----@param callback fun(items:table, errMsg:string) -function TradeQueryRequestsClass:FetchResults(itemHashes, queryId, callback) - local quantity_found = math.min(#itemHashes, self.maxFetchPerSearch) - local max_block_size = 10 - local items = {} - for fetch_block_start = 1, quantity_found, max_block_size do - local fetch_block_end = math.min(fetch_block_start + max_block_size - 1, quantity_found) - local param_item_hashes = table.concat({unpack(itemHashes, fetch_block_start, fetch_block_end)}, ",") - local fetch_url = self.hostName .. "api/trade2/fetch/"..param_item_hashes.."?query="..queryId - self:FetchResultBlock(fetch_url, function(itemBlock, errMsg) - if errMsg then - return callback(nil, errMsg) - end - for _, item in pairs(itemBlock) do - table.insert(items, item) - end - -- finished fetching item blocks - if #items >= quantity_found then - callback(items) - end - end) - end -end - ----Fetch details for paginated items ----@param url string ----@param callback fun(items: table, errMsg:string) -function TradeQueryRequestsClass:FetchResultBlock(url, callback) - table.insert(self.requestQueue["fetch"], { - url = url, - callback = function(response, errMsg) - if errMsg then - return callback(nil, errMsg) - end - local response, response_err = dkjson.decode(response) - if not response or not response.result then - if response_err then - errMsg = "JSON Parse Error: " .. (errMsg or "") - else - errMsg = "Failed to Get Trade Items: " .. (errMsg or "") - end - return callback(nil, errMsg) - end - local items = {} - for _, trade_entry in pairs(response.result) do - local item = trade_entry.item - local spirit - local armour - local evasion - local es - local charmSlots - local quality - local radius - local limit - local t_insert = table.insert - -- local catalystList = {"Abrasive", "Accelerating", "Fertile", "Imbued", "Intrinsic", "Noxious", "Prismatic", "Tempering", "Turbulent", "Unstable"} - - if item.properties then - for _, property in ipairs(item.properties) do - local name = escapeGGGString(property.name) - if name == "Armour" then - armour = property.values[1][1] - elseif name == "Evasion Rating" then - evasion = property.values[1][1] - elseif name == "Energy Shield" then - es = property.values[1][1] - elseif name == "Quality" then - quality = property.values[1][1]:sub(2, -2) -- remove + and % on quality value - elseif name == "Spirit" then - spirit = property.values[1][1] - elseif name == "Charm Slots" then - charmSlots = property.values[1][1] - -- elseif name == "Quality (Mana Modifiers)" then - -- catalyst quality stuff tbd it all needs reworking anyway as it has changed. - elseif name == "Radius" then - radius = property.values[1][1] - elseif name == "Limited to" then - limit = property.values[1][1] - end - end - end - - local rawLines = { } - t_insert(rawLines, "Rarity: " .. item.rarity) - -- item.name is empty when magic and full magic name is in typeLine but typeLine == baseType when rare. - if item.name ~= "" then - t_insert(rawLines, item.name) - end - t_insert(rawLines, item.typeLine) - - if charmSlots then - t_insert(rawLines, "Charm Slots: " .. charmSlots) - end - - if spirit then - t_insert(rawLines, "Spirit: " .. spirit) - end - - if armour then - t_insert(rawLines, "Armour: " .. armour) - end - if evasion then - t_insert(rawLines, "Evasion: " ..evasion) - end - if es then - t_insert(rawLines, "Energy Shield: " .. es) - end - - -- if self.catalyst and self.catalyst > 0 then - -- t_insert(rawLines, "Catalyst: " .. catalystList[self.catalyst]) - -- end - -- if self.catalystQuality then - -- t_insert(rawLines, "CatalystQuality: " .. self.catalystQuality) - -- end - - if item.ilvl then - t_insert(rawLines, "Item Level: " .. item.ilvl) - end - if quality then - t_insert(rawLines, "Quality: " .. quality) - end - if item.sockets then - local socketString = "" - for _, _ in ipairs(item.sockets) do - socketString = socketString .. "S " - end - socketString = socketString:gsub(" $", "") - t_insert(rawLines, "Sockets: " .. socketString) - end - - if item.requirements then - for _, requirement in ipairs(item.requirements) do - if requirement.name == "Level" then - t_insert(rawLines, "LevelReq: " .. requirement.values[1][1]) - end - end - end - - if radius then - t_insert(rawLines, "Radius: " .. radius) - end - if limit then - t_insert(rawLines, "Limited to: " .. limit) - end - - -- ensure these fields are initialised - item.enchantMods = item.enchantMods or { } - item.fracturedMods = item.fracturedMods or { } - item.desecratedMods = item.desecratedMods or { } - item.craftedMods = item.craftedMods or { } - item.runeMods = item.runeMods or { } - item.implicitMods = item.implicitMods or { } - item.explicitMods = item.explicitMods or { } - - local function processLine(modLine) - local s = "" - for flagName, flag in pairs(modLine.flags or {}) do - if flag then - s = s .. string.format("{%s}", flagName) - end - end - return s .. escapeGGGString(modLine.description) - end - t_insert(rawLines, "Implicits: " .. (#item.enchantMods + #item.runeMods + #item.implicitMods)) - for _, modLine in ipairs(item.enchantMods or {}) do - t_insert(rawLines, "{enchant}" .. processLine(modLine)) - end - for _, modLine in ipairs(item.runeMods or {}) do - t_insert(rawLines, "{enchant}{rune}" .. processLine(modLine)) - end - for _, modLine in ipairs(item.implicitMods or {}) do - t_insert(rawLines, processLine(modLine)) - end - for _, modLine in ipairs(item.explicitMods or {}) do - t_insert(rawLines, processLine(modLine)) - end - if item.mirrored then - t_insert(rawLines, "Mirrored") - end - if item.doubleCorrupted then - t_insert(rawLines, "Twice Corrupted") - elseif item.corrupted then - t_insert(rawLines, "Corrupted") - end - if item.sanctified then - t_insert(rawLines, "Sanctified") - end - - local pseudoMod = trade_entry.item.pseudoMods and trade_entry.item.pseudoMods[1] - local pseudoModLine = pseudoMod and (pseudoMod.description or pseudoMod) - table.insert(items, { - amount = trade_entry.listing.price.amount, - currency = trade_entry.listing.price.currency, - priceType = trade_entry.listing.price.type, - item_string = table.concat(rawLines, "\n"), - whisper = trade_entry.listing.whisper, - trader = trade_entry.listing.account.name, - weight = trade_entry.item.pseudoMods and pseudoModLine:match("Sum: (.+)") or "0", - id = trade_entry.id - }) - end - return callback(items) - end - }) -end - ----@param callback fun(items:table, errMsg:string, query: string?) -function TradeQueryRequestsClass:SearchWithURL(url, callback) - local subpath = url:match(self.hostName .. "trade2/search/(.+)$") - local paths = {} - for path in subpath:gmatch("[^/]+") do - table.insert(paths, path) - end - if #paths < 2 or #paths > 3 then - return callback(nil, "Invalid URL", nil) - end - local realm, league, queryId - if #paths == 3 then - realm = paths[1] - end - league = paths[#paths-1] - queryId = paths[#paths] - self:FetchSearchQuery(realm, league, queryId, function(query, errMsg) - if errMsg then - return callback(nil, errMsg, nil) - end - - -- update sorting on provided url to sort by weights. - local json_data = dkjson.decode(query) - if not json_data or json_data.error then - errMsg = json_data and json_data.error or "Failed to parse search query JSON" - end - if json_data.query.stats and json_data.query.stats[1] and json_data.query.stats[1].type == "weight" then - json_data.sort = {} - json_data.sort["statgroup.0"] = "desc" - else - json_data.sort = { price = "asc"} - end - query = dkjson.encode(json_data) - - self:SearchWithQuery(realm, league, query, function(items, searchErrMsg) - callback(items, searchErrMsg, query) - end) - end) -end - ----Fetch query data needed to perform the search ----@param queryId string ----@param league string ----@param callback fun(query:string, errMsg:string) -function TradeQueryRequestsClass:FetchSearchQuery(realm, league, queryId, callback) - local url = self:buildUrl(self.hostName .. "api/trade2/search", realm, league, queryId) - table.insert(self.requestQueue["search"], { - url = url, - callback = function(response, errMsg) - if errMsg then - return callback(nil, errMsg) - end - local json_data = dkjson.decode(response) - if not json_data or json_data.error then - errMsg = json_data and json_data.error or "Failed to get search query" - end - callback(response, errMsg) - end - }) -end - ---- Fetches the list of all available leagues using trade2 league API ----@param realm string ----@param callback fun(query:table, errMsg:string) -function TradeQueryRequestsClass:FetchLeagues(realm, callback) - local header = "Authorization: Bearer ".. (main.api.authToken or "") - launch:DownloadPage( - self.hostName .. "api/trade2/data/leagues", - function(response, errMsg) - if errMsg then - return callback({"Standard", "Hardcore"}, errMsg) - end - local json_data = dkjson.decode(response.body) - if not json_data or json_data.error then - errMsg = json_data and json_data.error or "Failed to parse trade leagues JSON" - end - local leagues = {} - for _, value in pairs(json_data.result) do - if value.realm == realm then - table.insert(leagues, value.id) - end - end - callback(leagues, errMsg) - end, - {header = header} - ) -end - ---- Build search and trade URLs with proper encoding ----@param root string ----@param realm string ----@param league string ----@param queryId string -function TradeQueryRequestsClass:buildUrl(root, realm, league, queryId) - local result = root - if realm and realm ~='pc' then - result = result .. "/" .. realm - end - local encodedLeague = league:gsub("[^%w%-%.%_%~]", function(c) - return string.format("%%%02X", string.byte(c)) - end):gsub(" ", "+") - result = result .. "/" .. encodedLeague - if queryId then - result = result .. "/" .. queryId - end - return result -end +-- Path of Building +-- +-- Module: Trade Query Requests +-- Handling trade api requests while respecting rate limits +-- + +local dkjson = require "dkjson" +local utils = LoadModule("Modules/Utils") + +---@class TradeQueryRequests +---@class TradeQueryRequests +local TradeQueryRequestsClass = newClass("TradeQueryRequests") + +function TradeQueryRequestsClass:TradeQueryRequests(rateLimiter) + self.maxFetchPerSearch = 10 + self.tradeQuery = tradeQuery + self.rateLimiter = rateLimiter or new("TradeQueryRateLimiter"):TradeQueryRateLimiter() + self.requestQueue = { + ["search"] = {}, + ["fetch"] = {}, + } + self.hostName = "https://www.pathofexile.com/" + return self +end + +---Main routine for processing request queue +--- @param onRateLimit fun(integer)? +function TradeQueryRequestsClass:ProcessQueue(onRateLimit) + for key, queue in pairs(self.requestQueue) do + if #queue > 0 then + local policy = self.rateLimiter:GetPolicyName(key) + local now = os.time() + local timeNext = self.rateLimiter:NextRequestTime(policy, now) + local timeLeft = timeNext - now + -- relay wait info to caller when actually waiting, and not just + -- getting a magic poe2 release date number + if onRateLimit and timeLeft > 1 and timeNext ~= 1956528000 then + onRateLimit(timeLeft) + end + if not (queue[1].retryTime and now < queue[1].retryTime) then + if now >= timeNext then + local request = table.remove(queue, 1) + local requestId = self.rateLimiter:InsertRequest(policy) + local onComplete = function(response, errMsg) + self.rateLimiter:FinishRequest(policy, requestId) + self.rateLimiter:UpdateFromHeader(response.header, policy) + if response.header:match("HTTP/[%d%.]+ (%d+)") == "429" then + local retryAfter = response.header:match("Retry%-After:%s+(%d+)") + retryAfter = retryAfter and tonumber(retryAfter) or 0 + request.attempts = (request.attempts or 0) + 1 + + local backoff = math.max(math.min(2 ^ request.attempts, 60), retryAfter) + request.retryTime = os.time() + backoff + table.insert(queue, 1, request) + -- optional callback with the backoff time when rate + -- limited to inform user + if onRateLimit then + onRateLimit(backoff) + end + return + end + if errMsg == "Response code: 401" and response.body:find("invalid_token") then + errMsg = errMsg .. "\nAuthorization is invalid. Please Re-Log and reset" + main.api:ResetDetails() + end + request.callback(response.body, errMsg, unpack(request.callbackParams or {})) + end + local header = "Content-Type: application/json" + if main.api.authToken then + header = header .."\nAuthorization: Bearer "..main.api.authToken + end + launch:DownloadPage(request.url, onComplete, { + header = header, + body = request.body, + }) + else + break + end + end + end + end +end + +---Performs search and fetches results +---@param league string +---@param query string +---@param callback fun(items:table, errMsg:string) +---@param params table @ params = { callbackQueryId = fun(queryId:string) } +function TradeQueryRequestsClass:SearchWithQuery(realm, league, query, callback, params) + params = params or {} + --ConPrintf("Query json: %s", query) + self:PerformSearch(realm, league, query, function(response, errMsg) + if params.callbackQueryId and response and response.id then + params.callbackQueryId(response.id) + end + if errMsg then + return callback(nil, errMsg) + end + self:FetchResults(response.result, response.id, callback) + end) +end + +---Performs search and fetches results, adjusting the query weight and repeating +---the search to fetch more items when the search cap (10k items) is reached +---@param league string +---@param query string +---@param callback fun(items:table, errMsg:string) +---@param params table @ params = { callbackQueryId = fun(queryId:string) } +function TradeQueryRequestsClass:SearchWithQueryWeightAdjusted(realm, league, query, callback, params) + params = params or {} + local previousSearchId = nil + local previousSearchItemIds = nil + local previousSearchItems = nil + -- Limit recursion to prevent potential loops + -- Each repeat is a leap of 10k items, normally we shouldn't need more than 1-2 steps anyways + local maxRecursion = 5 + local currentRecursion = 0 + local function performSearchCallback(response, errMsg) + currentRecursion = currentRecursion + 1 + if params.callbackQueryId and response and response.id then + params.callbackQueryId(response.id) + end + if errMsg and ((errMsg == "No Matching Results Found" and currentRecursion >= maxRecursion) or errMsg ~= "No Matching Results Found") then + return callback(nil, errMsg) + end + if (response.total > self.maxFetchPerSearch and response.total < 10000) or currentRecursion >= maxRecursion then + -- Search not clipped or max recursion reached, fetch results and finalize + if previousSearchItems and self.maxFetchPerSearch > response.total then + -- Not enough items in the last search, fill results from previous search + self:FetchResults(response.result, response.id, function(items, errMsg) + if errMsg then + return callback(nil, errMsg) + end + local fetchedItemIds = {} + local idSet = {} + for _, value in pairs(items) do + if not idSet[value.id] then + idSet[value.id] = true + table.insert(fetchedItemIds, value.id) + end + end + for _, value in pairs(previousSearchItems) do + if #items >= self.maxFetchPerSearch then + break + end + if not isValueInTable(fetchedItemIds, value.id) then + table.insert(items, value) + table.insert(fetchedItemIds, value.id) + end + end + local fillCount = self.maxFetchPerSearch - #items + if fillCount > 0 then + -- fill with previous search results + local unfetchedItemIds = {} + for _, value in pairs(previousSearchItemIds) do + if #unfetchedItemIds >= fillCount then + break + end + if not isValueInTable(fetchedItemIds, value) then + table.insert(unfetchedItemIds, value) + end + end + self:FetchResults(unfetchedItemIds, previousSearchId, function(newItems, errMsg) + if errMsg then + return callback(nil, errMsg) + end + items = tableConcat(items, newItems) + callback(items, errMsg) + end) + else + callback(items, errMsg) + end + end) + else + -- Search not clipped and result count satisfy maxFetchPerSearch, proceed normally + self:FetchResults(response.result, response.id, callback) + end + else + if response.total < self.maxFetchPerSearch then -- Less than maximum items retrieved lower weight to try and get more. + local queryJson = dkjson.decode(query) + queryJson.query.stats[1].value.min = queryJson.query.stats[1].value.min / 2 + query = dkjson.encode(queryJson) + self:PerformSearch(realm, league, query, performSearchCallback) + else -- Search clipped, fetch highest weight item, update query weight and repeat search + previousSearchItemIds = response.result + previousSearchId = response.id + local firstResultBatch = {unpack(response.result, 1, math.min(#response.result, 10))} + self:FetchResults(firstResultBatch, response.id, function(items, errMsg) + if errMsg then + return callback(nil, errMsg) + end + previousSearchItems = items + local highestWeight = items[1].weight + local queryJson = dkjson.decode(query) + queryJson.query.stats[1].value.min = (tonumber(highestWeight) + queryJson.query.stats[1].value.min) / 2 + query = dkjson.encode(queryJson) + self:PerformSearch(realm, league, query, performSearchCallback) + end) + end + end + end + self:PerformSearch(realm, league, query, performSearchCallback) +end + +---Perform search and run callback function on returned item hashes. +---Item info has to be fetched separately +---@param league string +---@param query string +---@param callback fun(response:table, errMsg:string) +function TradeQueryRequestsClass:PerformSearch(realm, league, query, callback) + table.insert(self.requestQueue["search"], { + url = self:buildUrl(self.hostName .. "api/trade2/search", realm, league), + body = query, + callback = function(response, errMsg) + if errMsg and not errMsg:find("Response code: 400") then + return callback(nil, errMsg) + end + local response = dkjson.decode(response) + if not response then + errMsg = "Failed to Get Trade response" + return callback(nil, errMsg) + end + if not response.result or #response.result == 0 then + if response.error then + if not (response.error.code and response.error.message) then + errMsg = "Encountered unknown error, check console for details." + ConPrintf("Unknown error: %s", utils.stringify(response.error)) + callback(response, errMsg) + end + if response.error.message:find("Logging in will increase this limit") then + errMsg = "Authorization is invalid. Please Re-Log and reset" + else + -- Report unhandled error + errMsg = "[ " .. response.error.code .. ": " .. response.error.message .. " ]" + end + else + ConPrintf("Found 0 results for %sapi/trade2/search/%s/%s", self.hostName, league, response.id) + errMsg = "No Matching Results Found" + end + return callback(response, errMsg) + end + callback(response, errMsg) + end, + }) +end + +---Fetch item details for itemHashes +---@param itemHashes string[] +---@param queryId string +---@param callback fun(items:table, errMsg:string) +function TradeQueryRequestsClass:FetchResults(itemHashes, queryId, callback) + local quantity_found = math.min(#itemHashes, self.maxFetchPerSearch) + local max_block_size = 10 + local items = {} + for fetch_block_start = 1, quantity_found, max_block_size do + local fetch_block_end = math.min(fetch_block_start + max_block_size - 1, quantity_found) + local param_item_hashes = table.concat({unpack(itemHashes, fetch_block_start, fetch_block_end)}, ",") + local fetch_url = self.hostName .. "api/trade2/fetch/"..param_item_hashes.."?query="..queryId + self:FetchResultBlock(fetch_url, function(itemBlock, errMsg) + if errMsg then + return callback(nil, errMsg) + end + for _, item in pairs(itemBlock) do + table.insert(items, item) + end + -- finished fetching item blocks + if #items >= quantity_found then + callback(items) + end + end) + end +end + +---Fetch details for paginated items +---@param url string +---@param callback fun(items: table, errMsg:string) +function TradeQueryRequestsClass:FetchResultBlock(url, callback) + table.insert(self.requestQueue["fetch"], { + url = url, + callback = function(response, errMsg) + if errMsg then + return callback(nil, errMsg) + end + local response, response_err = dkjson.decode(response) + if not response or not response.result then + if response_err then + errMsg = "JSON Parse Error: " .. (errMsg or "") + else + errMsg = "Failed to Get Trade Items: " .. (errMsg or "") + end + return callback(nil, errMsg) + end + local items = {} + for _, trade_entry in pairs(response.result) do + local item = trade_entry.item + local spirit + local armour + local evasion + local es + local charmSlots + local quality + local radius + local limit + local t_insert = table.insert + -- local catalystList = {"Abrasive", "Accelerating", "Fertile", "Imbued", "Intrinsic", "Noxious", "Prismatic", "Tempering", "Turbulent", "Unstable"} + + if item.properties then + for _, property in ipairs(item.properties) do + local name = escapeGGGString(property.name) + if name == "Armour" then + armour = property.values[1][1] + elseif name == "Evasion Rating" then + evasion = property.values[1][1] + elseif name == "Energy Shield" then + es = property.values[1][1] + elseif name == "Quality" then + quality = property.values[1][1]:sub(2, -2) -- remove + and % on quality value + elseif name == "Spirit" then + spirit = property.values[1][1] + elseif name == "Charm Slots" then + charmSlots = property.values[1][1] + -- elseif name == "Quality (Mana Modifiers)" then + -- catalyst quality stuff tbd it all needs reworking anyway as it has changed. + elseif name == "Radius" then + radius = property.values[1][1] + elseif name == "Limited to" then + limit = property.values[1][1] + end + end + end + + local rawLines = { } + t_insert(rawLines, "Rarity: " .. item.rarity) + -- item.name is empty when magic and full magic name is in typeLine but typeLine == baseType when rare. + if item.name ~= "" then + t_insert(rawLines, item.name) + end + t_insert(rawLines, item.typeLine) + + if charmSlots then + t_insert(rawLines, "Charm Slots: " .. charmSlots) + end + + if spirit then + t_insert(rawLines, "Spirit: " .. spirit) + end + + if armour then + t_insert(rawLines, "Armour: " .. armour) + end + if evasion then + t_insert(rawLines, "Evasion: " ..evasion) + end + if es then + t_insert(rawLines, "Energy Shield: " .. es) + end + + -- if self.catalyst and self.catalyst > 0 then + -- t_insert(rawLines, "Catalyst: " .. catalystList[self.catalyst]) + -- end + -- if self.catalystQuality then + -- t_insert(rawLines, "CatalystQuality: " .. self.catalystQuality) + -- end + + if item.ilvl then + t_insert(rawLines, "Item Level: " .. item.ilvl) + end + if quality then + t_insert(rawLines, "Quality: " .. quality) + end + if item.sockets then + local socketString = "" + for _, _ in ipairs(item.sockets) do + socketString = socketString .. "S " + end + socketString = socketString:gsub(" $", "") + t_insert(rawLines, "Sockets: " .. socketString) + end + + if item.requirements then + for _, requirement in ipairs(item.requirements) do + if requirement.name == "Level" then + t_insert(rawLines, "LevelReq: " .. requirement.values[1][1]) + end + end + end + + if radius then + t_insert(rawLines, "Radius: " .. radius) + end + if limit then + t_insert(rawLines, "Limited to: " .. limit) + end + + -- ensure these fields are initialised + item.enchantMods = item.enchantMods or { } + item.fracturedMods = item.fracturedMods or { } + item.desecratedMods = item.desecratedMods or { } + item.craftedMods = item.craftedMods or { } + item.runeMods = item.runeMods or { } + item.implicitMods = item.implicitMods or { } + item.explicitMods = item.explicitMods or { } + + local function processLine(modLine) + local s = "" + for flagName, flag in pairs(modLine.flags or {}) do + if flag then + s = s .. string.format("{%s}", flagName) + end + end + return s .. escapeGGGString(modLine.description) + end + t_insert(rawLines, "Implicits: " .. (#item.enchantMods + #item.runeMods + #item.implicitMods)) + for _, modLine in ipairs(item.enchantMods or {}) do + t_insert(rawLines, "{enchant}" .. processLine(modLine)) + end + for _, modLine in ipairs(item.runeMods or {}) do + t_insert(rawLines, "{enchant}{rune}" .. processLine(modLine)) + end + for _, modLine in ipairs(item.implicitMods or {}) do + t_insert(rawLines, processLine(modLine)) + end + for _, modLine in ipairs(item.explicitMods or {}) do + t_insert(rawLines, processLine(modLine)) + end + if item.mirrored then + t_insert(rawLines, "Mirrored") + end + if item.doubleCorrupted then + t_insert(rawLines, "Twice Corrupted") + elseif item.corrupted then + t_insert(rawLines, "Corrupted") + end + if item.sanctified then + t_insert(rawLines, "Sanctified") + end + + local pseudoMod = trade_entry.item.pseudoMods and trade_entry.item.pseudoMods[1] + local pseudoModLine = pseudoMod and (pseudoMod.description or pseudoMod) + table.insert(items, { + amount = trade_entry.listing.price.amount, + currency = trade_entry.listing.price.currency, + priceType = trade_entry.listing.price.type, + item_string = table.concat(rawLines, "\n"), + whisper = trade_entry.listing.whisper, + trader = trade_entry.listing.account.name, + weight = trade_entry.item.pseudoMods and pseudoModLine:match("Sum: (.+)") or "0", + id = trade_entry.id + }) + end + return callback(items) + end + }) +end + +---@param callback fun(items:table, errMsg:string, query: string?) +function TradeQueryRequestsClass:SearchWithURL(url, callback) + local prefix = self.hostName .. "trade2/search/" + if url:sub(1, #prefix) ~= prefix then + return callback(nil, "Invalid URL", nil) + end + local subpath = url:sub(#prefix + 1) + local paths = {} + for path in subpath:gmatch("[^/]+") do + table.insert(paths, path) + end + if #paths < 2 or #paths > 3 then + return callback(nil, "Invalid URL", nil) + end + local realm, league, queryId + if #paths == 3 then + realm = paths[1] + end + -- URL path segments are already escaped; buildUrl encodes the league again. + league = paths[#paths-1]:gsub("%%(%x%x)", function(hex) + return string.char(tonumber(hex, 16)) + end) + queryId = paths[#paths] + self:FetchSearchQuery(realm, league, queryId, function(query, errMsg) + if errMsg then + return callback(nil, errMsg, nil) + end + + -- update sorting on provided url to sort by weights. + local json_data = dkjson.decode(query) + if type(json_data) ~= "table" or json_data.error or type(json_data.query) ~= "table" then + return callback(nil, type(json_data) == "table" and json_data.error or "Failed to parse search query JSON", nil) + end + if json_data.query.stats and json_data.query.stats[1] and json_data.query.stats[1].type == "weight" then + json_data.sort = {} + json_data.sort["statgroup.0"] = "desc" + else + json_data.sort = { price = "asc"} + end + query = dkjson.encode(json_data) + + self:SearchWithQuery(realm, league, query, function(items, searchErrMsg) + callback(items, searchErrMsg, query) + end) + end) +end + +---Fetch query data needed to perform the search +---@param queryId string +---@param league string +---@param callback fun(query:string, errMsg:string) +function TradeQueryRequestsClass:FetchSearchQuery(realm, league, queryId, callback) + -- Browser share links can contain a gzip-compressed query instead of a saved ID. + if queryId:sub(1, 4) == "H4sI" then + local ok, query = pcall(function() + local compressed = require("base64").decode(queryId:gsub("-", "+"):gsub("_", "/")) + return LoadModule("Modules/TradeQueryDecode")(compressed) + end) + if not ok or not query then + return callback(nil, "Failed to decode compressed search query") + end + local data = dkjson.decode(query) + if type(data) ~= "table" then + return callback(nil, "Failed to parse compressed search query") + end + return callback(dkjson.encode({ query = data })) + end + local url = self:buildUrl(self.hostName .. "api/trade2/search", realm, league, queryId) + table.insert(self.requestQueue["search"], { + url = url, + callback = function(response, errMsg) + if errMsg then + return callback(nil, errMsg) + end + local json_data = dkjson.decode(response) + if not json_data or json_data.error then + errMsg = json_data and json_data.error or "Failed to get search query" + end + callback(response, errMsg) + end + }) +end + +--- Fetches the list of all available leagues using trade2 league API +---@param realm string +---@param callback fun(query:table, errMsg:string) +function TradeQueryRequestsClass:FetchLeagues(realm, callback) + local header = "Authorization: Bearer ".. (main.api.authToken or "") + launch:DownloadPage( + self.hostName .. "api/trade2/data/leagues", + function(response, errMsg) + if errMsg then + return callback({"Standard", "Hardcore"}, errMsg) + end + local json_data = dkjson.decode(response.body) + if not json_data or json_data.error then + errMsg = json_data and json_data.error or "Failed to parse trade leagues JSON" + end + local leagues = {} + for _, value in pairs(json_data.result) do + if value.realm == realm then + table.insert(leagues, value.id) + end + end + callback(leagues, errMsg) + end, + {header = header} + ) +end + +--- Build search and trade URLs with proper encoding +---@param root string +---@param realm string +---@param league string +---@param queryId string +function TradeQueryRequestsClass:buildUrl(root, realm, league, queryId) + local result = root + if realm and realm ~='pc' then + result = result .. "/" .. realm + end + local encodedLeague = league:gsub("[^%w%-%.%_%~]", function(c) + return string.format("%%%02X", string.byte(c)) + end):gsub(" ", "+") + result = result .. "/" .. encodedLeague + if queryId then + result = result .. "/" .. queryId + end + return result +end + From 763f0cd9b9beafacbab8668869472348b125729e Mon Sep 17 00:00:00 2001 From: Kanaxai Date: Mon, 7 Sep 2026 03:29:28 -0300 Subject: [PATCH 2/4] Add bounded gzip decoder for trade share queries --- src/Modules/TradeQueryDecode.lua | 50 ++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 src/Modules/TradeQueryDecode.lua diff --git a/src/Modules/TradeQueryDecode.lua b/src/Modules/TradeQueryDecode.lua new file mode 100644 index 0000000000..d6ccc5b2cc --- /dev/null +++ b/src/Modules/TradeQueryDecode.lua @@ -0,0 +1,50 @@ +-- Decode gzip-compressed trade website share queries. +-- SimpleGraphic's Inflate only supports zlib streams, so use zlib's gzip mode. +local ffi = require("ffi") +if not pcall(ffi.typeof, "pob_trade_z_stream") then +ffi.cdef[[ +typedef struct { + const unsigned char *next_in; + unsigned int avail_in; + unsigned long total_in; + unsigned char *next_out; + unsigned int avail_out; + unsigned long total_out; + const char *msg; + void *state; + void *(*zalloc)(void *, unsigned int, unsigned int); + void (*zfree)(void *, void *); + void *opaque; + int data_type; + unsigned long adler; + unsigned long reserved; +} pob_trade_z_stream; +const char *zlibVersion(void); +int inflateInit2_(pob_trade_z_stream *, int, const char *, int); +int inflate(pob_trade_z_stream *, int); +int inflateEnd(pob_trade_z_stream *); +]] +end +local zlib = ffi.load(ffi.os == "Windows" and "zlib1" or "z") + +return function(compressed) + -- Trade queries are small; cap expansion to avoid unbounded allocations. + local capacity = 1024 * 1024 + local output = ffi.new("unsigned char[?]", capacity) + local stream = ffi.new("pob_trade_z_stream[1]") + stream[0].next_in = compressed + stream[0].avail_in = #compressed + stream[0].next_out = output + stream[0].avail_out = capacity + if zlib.inflateInit2_(stream, 31, zlib.zlibVersion(), ffi.sizeof(stream[0])) ~= 0 then + return nil + end + local status = zlib.inflate(stream, 4) -- Z_FINISH + local length = tonumber(stream[0].total_out) + local remaining = stream[0].avail_in + zlib.inflateEnd(stream) + if status ~= 1 or remaining ~= 0 then -- Z_STREAM_END + return nil + end + return ffi.string(output, length) +end From 7b388ab760b82f48bcf2e4d4854292da68e565f2 Mon Sep 17 00:00:00 2001 From: Kanaxai Date: Mon, 7 Sep 2026 03:29:56 -0300 Subject: [PATCH 3/4] Cover encoded leagues and compressed trade URLs --- spec/System/TestTradeQueryRequests_spec.lua | 70 ++++++++++++++++++++- 1 file changed, 69 insertions(+), 1 deletion(-) diff --git a/spec/System/TestTradeQueryRequests_spec.lua b/spec/System/TestTradeQueryRequests_spec.lua index 8fb665e871..07b89c5238 100644 --- a/spec/System/TestTradeQueryRequests_spec.lua +++ b/spec/System/TestTradeQueryRequests_spec.lua @@ -228,4 +228,72 @@ Strict-Transport-Security: max-age=63115200; includeSubDomains; preload]] requests.FetchResultBlock = orig_fetchBlock end) end) -end) \ No newline at end of file +end) + +describe("TradeQueryRequests URL searches", function() + local requests + before_each(function() + requests = new("TradeQueryRequests"):TradeQueryRequests() + end) + + it("encodes the league once for both query lookup and search", function() + requests:SearchWithURL("https://www.pathofexile.com/trade2/search/poe2/Forbidden%20Rites/example", function() end) + local lookup = table.remove(requests.requestQueue.search, 1) + assert.are.equal("https://www.pathofexile.com/api/trade2/search/poe2/Forbidden%20Rites/example", lookup.url) + lookup.callback('{"query":{"stats":[]}}') + local search = requests.requestQueue.search[1] + assert.are.equal("https://www.pathofexile.com/api/trade2/search/poe2/Forbidden%20Rites", search.url) + assert.are.same({ price = "asc" }, require("dkjson").decode(search.body).sort) + end) + + it("preserves escaped percent signs and slashes in league names", function() + requests:SearchWithURL("https://www.pathofexile.com/trade2/search/poe2/Test%2520%2FLeague/example", function() end) + assert.are.equal("https://www.pathofexile.com/api/trade2/search/poe2/Test%2520%2FLeague/example", requests.requestQueue.search[1].url) + end) + + it("still accepts legacy URLs without a realm", function() + requests:SearchWithURL("https://www.pathofexile.com/trade2/search/Standard/example", function() end) + assert.are.equal("https://www.pathofexile.com/api/trade2/search/Standard/example", requests.requestQueue.search[1].url) + end) + + it("rejects unrelated URLs without throwing or queueing a request", function() + local errorMessage + requests:SearchWithURL("https://example.com/trade2/search/Standard/example", function(_, err) + errorMessage = err + end) + assert.are.equal("Invalid URL", errorMessage) + assert.are.equal(0, #requests.requestQueue.search) + end) + + it("reports malformed query responses without starting a search", function() + local errorMessage + requests:SearchWithURL("https://www.pathofexile.com/trade2/search/Standard/example", function(_, err) + errorMessage = err + end) + local lookup = table.remove(requests.requestQueue.search, 1) + lookup.callback('{}') + assert.are.equal("Failed to parse search query JSON", errorMessage) + assert.are.equal(0, #requests.requestQueue.search) + end) + it("submits compressed browser queries directly, including repeated searches", function() + local payload = "H4sIAAAAAAAAA42U62rkMAyF38W_hyDLF0nzKksp3ozbGtJMmksvlHn3VaalTSGm-yPEIcdfjk4kv5u70s15nMzx3Yz56XbzeCpT-tvlkznepW7Kh620e-7W22N6NUcPl8vlYOYxnfJ2_2Y5jKXNXxvsh_5tqMnbNOf78_i2rs_DXM69OZqXnIZz30xtHuYxGyWMaSzzT1F_7pe-PC36_vqRaU6zIv9s8LouWpPJr0NX2jI3q-YWbUQQ8gHNwTynbrnafcnl_mE2R-eoYYgOA3FAUvIuxJLwKmG_BwkNB6AYmCJa5BoEOXgKVmAXYmMj6MSFCBY81iDsRZg8xhqDick6dQNVIzZoHALRyx4EoWFxguQ4sPPVapwncDHybq6rE1IMgY1kN4y-fUj9f6Ya0YtHlUSplqJpSNB6az_GRyC9nLJczcavDLKktYoI1o2gtoBFsrAbRwMOmLTbPKLEGsRB0MDQQ9iDQENiI8P6KUe2SvHA4C1Hv1sPNFGs0w5jr6Ggr4TyqxVmG5i13x2QduvNx9xfx_mq2ex6LDrBsSEQzxyJBD5b4nt0v7en_mRW2upimX6cAVNul3E9ufQM-AfqJq6d4AQAAA" + for i = 1, 2 do + requests:SearchWithURL("https://www.pathofexile.com/trade2/search/poe2/Forbidden%20Rites/" .. payload, function() end) + local search = table.remove(requests.requestQueue.search, 1) + assert.are.equal("https://www.pathofexile.com/api/trade2/search/poe2/Forbidden%20Rites", search.url) + local query = require("dkjson").decode(search.body) + assert.are.equal("weight", query.query.stats[1].type) + assert.are.equal("desc", query.sort["statgroup.0"]) + end + end) + + it("rejects damaged compressed queries without making a request", function() + local errorMessage + requests:SearchWithURL("https://www.pathofexile.com/trade2/search/poe2/Standard/H4sIAAAA", function(_, err) + errorMessage = err + end) + assert.are.equal("Failed to decode compressed search query", errorMessage) + assert.are.equal(0, #requests.requestQueue.search) + end) + +end) From 7e3e65533e07c27b9856fb909a262b92cb697789 Mon Sep 17 00:00:00 2001 From: Kanaxai Date: Mon, 7 Sep 2026 03:30:24 -0300 Subject: [PATCH 4/4] Preserve upstream line endings --- src/Classes/TradeQueryRequests.lua | 1169 ++++++++++++++-------------- 1 file changed, 584 insertions(+), 585 deletions(-) diff --git a/src/Classes/TradeQueryRequests.lua b/src/Classes/TradeQueryRequests.lua index 7cbeb47833..5322ee0c5d 100644 --- a/src/Classes/TradeQueryRequests.lua +++ b/src/Classes/TradeQueryRequests.lua @@ -1,585 +1,584 @@ --- Path of Building --- --- Module: Trade Query Requests --- Handling trade api requests while respecting rate limits --- - -local dkjson = require "dkjson" -local utils = LoadModule("Modules/Utils") - ----@class TradeQueryRequests ----@class TradeQueryRequests -local TradeQueryRequestsClass = newClass("TradeQueryRequests") - -function TradeQueryRequestsClass:TradeQueryRequests(rateLimiter) - self.maxFetchPerSearch = 10 - self.tradeQuery = tradeQuery - self.rateLimiter = rateLimiter or new("TradeQueryRateLimiter"):TradeQueryRateLimiter() - self.requestQueue = { - ["search"] = {}, - ["fetch"] = {}, - } - self.hostName = "https://www.pathofexile.com/" - return self -end - ----Main routine for processing request queue ---- @param onRateLimit fun(integer)? -function TradeQueryRequestsClass:ProcessQueue(onRateLimit) - for key, queue in pairs(self.requestQueue) do - if #queue > 0 then - local policy = self.rateLimiter:GetPolicyName(key) - local now = os.time() - local timeNext = self.rateLimiter:NextRequestTime(policy, now) - local timeLeft = timeNext - now - -- relay wait info to caller when actually waiting, and not just - -- getting a magic poe2 release date number - if onRateLimit and timeLeft > 1 and timeNext ~= 1956528000 then - onRateLimit(timeLeft) - end - if not (queue[1].retryTime and now < queue[1].retryTime) then - if now >= timeNext then - local request = table.remove(queue, 1) - local requestId = self.rateLimiter:InsertRequest(policy) - local onComplete = function(response, errMsg) - self.rateLimiter:FinishRequest(policy, requestId) - self.rateLimiter:UpdateFromHeader(response.header, policy) - if response.header:match("HTTP/[%d%.]+ (%d+)") == "429" then - local retryAfter = response.header:match("Retry%-After:%s+(%d+)") - retryAfter = retryAfter and tonumber(retryAfter) or 0 - request.attempts = (request.attempts or 0) + 1 - - local backoff = math.max(math.min(2 ^ request.attempts, 60), retryAfter) - request.retryTime = os.time() + backoff - table.insert(queue, 1, request) - -- optional callback with the backoff time when rate - -- limited to inform user - if onRateLimit then - onRateLimit(backoff) - end - return - end - if errMsg == "Response code: 401" and response.body:find("invalid_token") then - errMsg = errMsg .. "\nAuthorization is invalid. Please Re-Log and reset" - main.api:ResetDetails() - end - request.callback(response.body, errMsg, unpack(request.callbackParams or {})) - end - local header = "Content-Type: application/json" - if main.api.authToken then - header = header .."\nAuthorization: Bearer "..main.api.authToken - end - launch:DownloadPage(request.url, onComplete, { - header = header, - body = request.body, - }) - else - break - end - end - end - end -end - ----Performs search and fetches results ----@param league string ----@param query string ----@param callback fun(items:table, errMsg:string) ----@param params table @ params = { callbackQueryId = fun(queryId:string) } -function TradeQueryRequestsClass:SearchWithQuery(realm, league, query, callback, params) - params = params or {} - --ConPrintf("Query json: %s", query) - self:PerformSearch(realm, league, query, function(response, errMsg) - if params.callbackQueryId and response and response.id then - params.callbackQueryId(response.id) - end - if errMsg then - return callback(nil, errMsg) - end - self:FetchResults(response.result, response.id, callback) - end) -end - ----Performs search and fetches results, adjusting the query weight and repeating ----the search to fetch more items when the search cap (10k items) is reached ----@param league string ----@param query string ----@param callback fun(items:table, errMsg:string) ----@param params table @ params = { callbackQueryId = fun(queryId:string) } -function TradeQueryRequestsClass:SearchWithQueryWeightAdjusted(realm, league, query, callback, params) - params = params or {} - local previousSearchId = nil - local previousSearchItemIds = nil - local previousSearchItems = nil - -- Limit recursion to prevent potential loops - -- Each repeat is a leap of 10k items, normally we shouldn't need more than 1-2 steps anyways - local maxRecursion = 5 - local currentRecursion = 0 - local function performSearchCallback(response, errMsg) - currentRecursion = currentRecursion + 1 - if params.callbackQueryId and response and response.id then - params.callbackQueryId(response.id) - end - if errMsg and ((errMsg == "No Matching Results Found" and currentRecursion >= maxRecursion) or errMsg ~= "No Matching Results Found") then - return callback(nil, errMsg) - end - if (response.total > self.maxFetchPerSearch and response.total < 10000) or currentRecursion >= maxRecursion then - -- Search not clipped or max recursion reached, fetch results and finalize - if previousSearchItems and self.maxFetchPerSearch > response.total then - -- Not enough items in the last search, fill results from previous search - self:FetchResults(response.result, response.id, function(items, errMsg) - if errMsg then - return callback(nil, errMsg) - end - local fetchedItemIds = {} - local idSet = {} - for _, value in pairs(items) do - if not idSet[value.id] then - idSet[value.id] = true - table.insert(fetchedItemIds, value.id) - end - end - for _, value in pairs(previousSearchItems) do - if #items >= self.maxFetchPerSearch then - break - end - if not isValueInTable(fetchedItemIds, value.id) then - table.insert(items, value) - table.insert(fetchedItemIds, value.id) - end - end - local fillCount = self.maxFetchPerSearch - #items - if fillCount > 0 then - -- fill with previous search results - local unfetchedItemIds = {} - for _, value in pairs(previousSearchItemIds) do - if #unfetchedItemIds >= fillCount then - break - end - if not isValueInTable(fetchedItemIds, value) then - table.insert(unfetchedItemIds, value) - end - end - self:FetchResults(unfetchedItemIds, previousSearchId, function(newItems, errMsg) - if errMsg then - return callback(nil, errMsg) - end - items = tableConcat(items, newItems) - callback(items, errMsg) - end) - else - callback(items, errMsg) - end - end) - else - -- Search not clipped and result count satisfy maxFetchPerSearch, proceed normally - self:FetchResults(response.result, response.id, callback) - end - else - if response.total < self.maxFetchPerSearch then -- Less than maximum items retrieved lower weight to try and get more. - local queryJson = dkjson.decode(query) - queryJson.query.stats[1].value.min = queryJson.query.stats[1].value.min / 2 - query = dkjson.encode(queryJson) - self:PerformSearch(realm, league, query, performSearchCallback) - else -- Search clipped, fetch highest weight item, update query weight and repeat search - previousSearchItemIds = response.result - previousSearchId = response.id - local firstResultBatch = {unpack(response.result, 1, math.min(#response.result, 10))} - self:FetchResults(firstResultBatch, response.id, function(items, errMsg) - if errMsg then - return callback(nil, errMsg) - end - previousSearchItems = items - local highestWeight = items[1].weight - local queryJson = dkjson.decode(query) - queryJson.query.stats[1].value.min = (tonumber(highestWeight) + queryJson.query.stats[1].value.min) / 2 - query = dkjson.encode(queryJson) - self:PerformSearch(realm, league, query, performSearchCallback) - end) - end - end - end - self:PerformSearch(realm, league, query, performSearchCallback) -end - ----Perform search and run callback function on returned item hashes. ----Item info has to be fetched separately ----@param league string ----@param query string ----@param callback fun(response:table, errMsg:string) -function TradeQueryRequestsClass:PerformSearch(realm, league, query, callback) - table.insert(self.requestQueue["search"], { - url = self:buildUrl(self.hostName .. "api/trade2/search", realm, league), - body = query, - callback = function(response, errMsg) - if errMsg and not errMsg:find("Response code: 400") then - return callback(nil, errMsg) - end - local response = dkjson.decode(response) - if not response then - errMsg = "Failed to Get Trade response" - return callback(nil, errMsg) - end - if not response.result or #response.result == 0 then - if response.error then - if not (response.error.code and response.error.message) then - errMsg = "Encountered unknown error, check console for details." - ConPrintf("Unknown error: %s", utils.stringify(response.error)) - callback(response, errMsg) - end - if response.error.message:find("Logging in will increase this limit") then - errMsg = "Authorization is invalid. Please Re-Log and reset" - else - -- Report unhandled error - errMsg = "[ " .. response.error.code .. ": " .. response.error.message .. " ]" - end - else - ConPrintf("Found 0 results for %sapi/trade2/search/%s/%s", self.hostName, league, response.id) - errMsg = "No Matching Results Found" - end - return callback(response, errMsg) - end - callback(response, errMsg) - end, - }) -end - ----Fetch item details for itemHashes ----@param itemHashes string[] ----@param queryId string ----@param callback fun(items:table, errMsg:string) -function TradeQueryRequestsClass:FetchResults(itemHashes, queryId, callback) - local quantity_found = math.min(#itemHashes, self.maxFetchPerSearch) - local max_block_size = 10 - local items = {} - for fetch_block_start = 1, quantity_found, max_block_size do - local fetch_block_end = math.min(fetch_block_start + max_block_size - 1, quantity_found) - local param_item_hashes = table.concat({unpack(itemHashes, fetch_block_start, fetch_block_end)}, ",") - local fetch_url = self.hostName .. "api/trade2/fetch/"..param_item_hashes.."?query="..queryId - self:FetchResultBlock(fetch_url, function(itemBlock, errMsg) - if errMsg then - return callback(nil, errMsg) - end - for _, item in pairs(itemBlock) do - table.insert(items, item) - end - -- finished fetching item blocks - if #items >= quantity_found then - callback(items) - end - end) - end -end - ----Fetch details for paginated items ----@param url string ----@param callback fun(items: table, errMsg:string) -function TradeQueryRequestsClass:FetchResultBlock(url, callback) - table.insert(self.requestQueue["fetch"], { - url = url, - callback = function(response, errMsg) - if errMsg then - return callback(nil, errMsg) - end - local response, response_err = dkjson.decode(response) - if not response or not response.result then - if response_err then - errMsg = "JSON Parse Error: " .. (errMsg or "") - else - errMsg = "Failed to Get Trade Items: " .. (errMsg or "") - end - return callback(nil, errMsg) - end - local items = {} - for _, trade_entry in pairs(response.result) do - local item = trade_entry.item - local spirit - local armour - local evasion - local es - local charmSlots - local quality - local radius - local limit - local t_insert = table.insert - -- local catalystList = {"Abrasive", "Accelerating", "Fertile", "Imbued", "Intrinsic", "Noxious", "Prismatic", "Tempering", "Turbulent", "Unstable"} - - if item.properties then - for _, property in ipairs(item.properties) do - local name = escapeGGGString(property.name) - if name == "Armour" then - armour = property.values[1][1] - elseif name == "Evasion Rating" then - evasion = property.values[1][1] - elseif name == "Energy Shield" then - es = property.values[1][1] - elseif name == "Quality" then - quality = property.values[1][1]:sub(2, -2) -- remove + and % on quality value - elseif name == "Spirit" then - spirit = property.values[1][1] - elseif name == "Charm Slots" then - charmSlots = property.values[1][1] - -- elseif name == "Quality (Mana Modifiers)" then - -- catalyst quality stuff tbd it all needs reworking anyway as it has changed. - elseif name == "Radius" then - radius = property.values[1][1] - elseif name == "Limited to" then - limit = property.values[1][1] - end - end - end - - local rawLines = { } - t_insert(rawLines, "Rarity: " .. item.rarity) - -- item.name is empty when magic and full magic name is in typeLine but typeLine == baseType when rare. - if item.name ~= "" then - t_insert(rawLines, item.name) - end - t_insert(rawLines, item.typeLine) - - if charmSlots then - t_insert(rawLines, "Charm Slots: " .. charmSlots) - end - - if spirit then - t_insert(rawLines, "Spirit: " .. spirit) - end - - if armour then - t_insert(rawLines, "Armour: " .. armour) - end - if evasion then - t_insert(rawLines, "Evasion: " ..evasion) - end - if es then - t_insert(rawLines, "Energy Shield: " .. es) - end - - -- if self.catalyst and self.catalyst > 0 then - -- t_insert(rawLines, "Catalyst: " .. catalystList[self.catalyst]) - -- end - -- if self.catalystQuality then - -- t_insert(rawLines, "CatalystQuality: " .. self.catalystQuality) - -- end - - if item.ilvl then - t_insert(rawLines, "Item Level: " .. item.ilvl) - end - if quality then - t_insert(rawLines, "Quality: " .. quality) - end - if item.sockets then - local socketString = "" - for _, _ in ipairs(item.sockets) do - socketString = socketString .. "S " - end - socketString = socketString:gsub(" $", "") - t_insert(rawLines, "Sockets: " .. socketString) - end - - if item.requirements then - for _, requirement in ipairs(item.requirements) do - if requirement.name == "Level" then - t_insert(rawLines, "LevelReq: " .. requirement.values[1][1]) - end - end - end - - if radius then - t_insert(rawLines, "Radius: " .. radius) - end - if limit then - t_insert(rawLines, "Limited to: " .. limit) - end - - -- ensure these fields are initialised - item.enchantMods = item.enchantMods or { } - item.fracturedMods = item.fracturedMods or { } - item.desecratedMods = item.desecratedMods or { } - item.craftedMods = item.craftedMods or { } - item.runeMods = item.runeMods or { } - item.implicitMods = item.implicitMods or { } - item.explicitMods = item.explicitMods or { } - - local function processLine(modLine) - local s = "" - for flagName, flag in pairs(modLine.flags or {}) do - if flag then - s = s .. string.format("{%s}", flagName) - end - end - return s .. escapeGGGString(modLine.description) - end - t_insert(rawLines, "Implicits: " .. (#item.enchantMods + #item.runeMods + #item.implicitMods)) - for _, modLine in ipairs(item.enchantMods or {}) do - t_insert(rawLines, "{enchant}" .. processLine(modLine)) - end - for _, modLine in ipairs(item.runeMods or {}) do - t_insert(rawLines, "{enchant}{rune}" .. processLine(modLine)) - end - for _, modLine in ipairs(item.implicitMods or {}) do - t_insert(rawLines, processLine(modLine)) - end - for _, modLine in ipairs(item.explicitMods or {}) do - t_insert(rawLines, processLine(modLine)) - end - if item.mirrored then - t_insert(rawLines, "Mirrored") - end - if item.doubleCorrupted then - t_insert(rawLines, "Twice Corrupted") - elseif item.corrupted then - t_insert(rawLines, "Corrupted") - end - if item.sanctified then - t_insert(rawLines, "Sanctified") - end - - local pseudoMod = trade_entry.item.pseudoMods and trade_entry.item.pseudoMods[1] - local pseudoModLine = pseudoMod and (pseudoMod.description or pseudoMod) - table.insert(items, { - amount = trade_entry.listing.price.amount, - currency = trade_entry.listing.price.currency, - priceType = trade_entry.listing.price.type, - item_string = table.concat(rawLines, "\n"), - whisper = trade_entry.listing.whisper, - trader = trade_entry.listing.account.name, - weight = trade_entry.item.pseudoMods and pseudoModLine:match("Sum: (.+)") or "0", - id = trade_entry.id - }) - end - return callback(items) - end - }) -end - ----@param callback fun(items:table, errMsg:string, query: string?) -function TradeQueryRequestsClass:SearchWithURL(url, callback) - local prefix = self.hostName .. "trade2/search/" - if url:sub(1, #prefix) ~= prefix then - return callback(nil, "Invalid URL", nil) - end - local subpath = url:sub(#prefix + 1) - local paths = {} - for path in subpath:gmatch("[^/]+") do - table.insert(paths, path) - end - if #paths < 2 or #paths > 3 then - return callback(nil, "Invalid URL", nil) - end - local realm, league, queryId - if #paths == 3 then - realm = paths[1] - end - -- URL path segments are already escaped; buildUrl encodes the league again. - league = paths[#paths-1]:gsub("%%(%x%x)", function(hex) - return string.char(tonumber(hex, 16)) - end) - queryId = paths[#paths] - self:FetchSearchQuery(realm, league, queryId, function(query, errMsg) - if errMsg then - return callback(nil, errMsg, nil) - end - - -- update sorting on provided url to sort by weights. - local json_data = dkjson.decode(query) - if type(json_data) ~= "table" or json_data.error or type(json_data.query) ~= "table" then - return callback(nil, type(json_data) == "table" and json_data.error or "Failed to parse search query JSON", nil) - end - if json_data.query.stats and json_data.query.stats[1] and json_data.query.stats[1].type == "weight" then - json_data.sort = {} - json_data.sort["statgroup.0"] = "desc" - else - json_data.sort = { price = "asc"} - end - query = dkjson.encode(json_data) - - self:SearchWithQuery(realm, league, query, function(items, searchErrMsg) - callback(items, searchErrMsg, query) - end) - end) -end - ----Fetch query data needed to perform the search ----@param queryId string ----@param league string ----@param callback fun(query:string, errMsg:string) -function TradeQueryRequestsClass:FetchSearchQuery(realm, league, queryId, callback) - -- Browser share links can contain a gzip-compressed query instead of a saved ID. - if queryId:sub(1, 4) == "H4sI" then - local ok, query = pcall(function() - local compressed = require("base64").decode(queryId:gsub("-", "+"):gsub("_", "/")) - return LoadModule("Modules/TradeQueryDecode")(compressed) - end) - if not ok or not query then - return callback(nil, "Failed to decode compressed search query") - end - local data = dkjson.decode(query) - if type(data) ~= "table" then - return callback(nil, "Failed to parse compressed search query") - end - return callback(dkjson.encode({ query = data })) - end - local url = self:buildUrl(self.hostName .. "api/trade2/search", realm, league, queryId) - table.insert(self.requestQueue["search"], { - url = url, - callback = function(response, errMsg) - if errMsg then - return callback(nil, errMsg) - end - local json_data = dkjson.decode(response) - if not json_data or json_data.error then - errMsg = json_data and json_data.error or "Failed to get search query" - end - callback(response, errMsg) - end - }) -end - ---- Fetches the list of all available leagues using trade2 league API ----@param realm string ----@param callback fun(query:table, errMsg:string) -function TradeQueryRequestsClass:FetchLeagues(realm, callback) - local header = "Authorization: Bearer ".. (main.api.authToken or "") - launch:DownloadPage( - self.hostName .. "api/trade2/data/leagues", - function(response, errMsg) - if errMsg then - return callback({"Standard", "Hardcore"}, errMsg) - end - local json_data = dkjson.decode(response.body) - if not json_data or json_data.error then - errMsg = json_data and json_data.error or "Failed to parse trade leagues JSON" - end - local leagues = {} - for _, value in pairs(json_data.result) do - if value.realm == realm then - table.insert(leagues, value.id) - end - end - callback(leagues, errMsg) - end, - {header = header} - ) -end - ---- Build search and trade URLs with proper encoding ----@param root string ----@param realm string ----@param league string ----@param queryId string -function TradeQueryRequestsClass:buildUrl(root, realm, league, queryId) - local result = root - if realm and realm ~='pc' then - result = result .. "/" .. realm - end - local encodedLeague = league:gsub("[^%w%-%.%_%~]", function(c) - return string.format("%%%02X", string.byte(c)) - end):gsub(" ", "+") - result = result .. "/" .. encodedLeague - if queryId then - result = result .. "/" .. queryId - end - return result -end - +-- Path of Building +-- +-- Module: Trade Query Requests +-- Handling trade api requests while respecting rate limits +-- + +local dkjson = require "dkjson" +local utils = LoadModule("Modules/Utils") + +---@class TradeQueryRequests +---@class TradeQueryRequests +local TradeQueryRequestsClass = newClass("TradeQueryRequests") + +function TradeQueryRequestsClass:TradeQueryRequests(rateLimiter) + self.maxFetchPerSearch = 10 + self.tradeQuery = tradeQuery + self.rateLimiter = rateLimiter or new("TradeQueryRateLimiter"):TradeQueryRateLimiter() + self.requestQueue = { + ["search"] = {}, + ["fetch"] = {}, + } + self.hostName = "https://www.pathofexile.com/" + return self +end + +---Main routine for processing request queue +--- @param onRateLimit fun(integer)? +function TradeQueryRequestsClass:ProcessQueue(onRateLimit) + for key, queue in pairs(self.requestQueue) do + if #queue > 0 then + local policy = self.rateLimiter:GetPolicyName(key) + local now = os.time() + local timeNext = self.rateLimiter:NextRequestTime(policy, now) + local timeLeft = timeNext - now + -- relay wait info to caller when actually waiting, and not just + -- getting a magic poe2 release date number + if onRateLimit and timeLeft > 1 and timeNext ~= 1956528000 then + onRateLimit(timeLeft) + end + if not (queue[1].retryTime and now < queue[1].retryTime) then + if now >= timeNext then + local request = table.remove(queue, 1) + local requestId = self.rateLimiter:InsertRequest(policy) + local onComplete = function(response, errMsg) + self.rateLimiter:FinishRequest(policy, requestId) + self.rateLimiter:UpdateFromHeader(response.header, policy) + if response.header:match("HTTP/[%d%.]+ (%d+)") == "429" then + local retryAfter = response.header:match("Retry%-After:%s+(%d+)") + retryAfter = retryAfter and tonumber(retryAfter) or 0 + request.attempts = (request.attempts or 0) + 1 + + local backoff = math.max(math.min(2 ^ request.attempts, 60), retryAfter) + request.retryTime = os.time() + backoff + table.insert(queue, 1, request) + -- optional callback with the backoff time when rate + -- limited to inform user + if onRateLimit then + onRateLimit(backoff) + end + return + end + if errMsg == "Response code: 401" and response.body:find("invalid_token") then + errMsg = errMsg .. "\nAuthorization is invalid. Please Re-Log and reset" + main.api:ResetDetails() + end + request.callback(response.body, errMsg, unpack(request.callbackParams or {})) + end + local header = "Content-Type: application/json" + if main.api.authToken then + header = header .."\nAuthorization: Bearer "..main.api.authToken + end + launch:DownloadPage(request.url, onComplete, { + header = header, + body = request.body, + }) + else + break + end + end + end + end +end + +---Performs search and fetches results +---@param league string +---@param query string +---@param callback fun(items:table, errMsg:string) +---@param params table @ params = { callbackQueryId = fun(queryId:string) } +function TradeQueryRequestsClass:SearchWithQuery(realm, league, query, callback, params) + params = params or {} + --ConPrintf("Query json: %s", query) + self:PerformSearch(realm, league, query, function(response, errMsg) + if params.callbackQueryId and response and response.id then + params.callbackQueryId(response.id) + end + if errMsg then + return callback(nil, errMsg) + end + self:FetchResults(response.result, response.id, callback) + end) +end + +---Performs search and fetches results, adjusting the query weight and repeating +---the search to fetch more items when the search cap (10k items) is reached +---@param league string +---@param query string +---@param callback fun(items:table, errMsg:string) +---@param params table @ params = { callbackQueryId = fun(queryId:string) } +function TradeQueryRequestsClass:SearchWithQueryWeightAdjusted(realm, league, query, callback, params) + params = params or {} + local previousSearchId = nil + local previousSearchItemIds = nil + local previousSearchItems = nil + -- Limit recursion to prevent potential loops + -- Each repeat is a leap of 10k items, normally we shouldn't need more than 1-2 steps anyways + local maxRecursion = 5 + local currentRecursion = 0 + local function performSearchCallback(response, errMsg) + currentRecursion = currentRecursion + 1 + if params.callbackQueryId and response and response.id then + params.callbackQueryId(response.id) + end + if errMsg and ((errMsg == "No Matching Results Found" and currentRecursion >= maxRecursion) or errMsg ~= "No Matching Results Found") then + return callback(nil, errMsg) + end + if (response.total > self.maxFetchPerSearch and response.total < 10000) or currentRecursion >= maxRecursion then + -- Search not clipped or max recursion reached, fetch results and finalize + if previousSearchItems and self.maxFetchPerSearch > response.total then + -- Not enough items in the last search, fill results from previous search + self:FetchResults(response.result, response.id, function(items, errMsg) + if errMsg then + return callback(nil, errMsg) + end + local fetchedItemIds = {} + local idSet = {} + for _, value in pairs(items) do + if not idSet[value.id] then + idSet[value.id] = true + table.insert(fetchedItemIds, value.id) + end + end + for _, value in pairs(previousSearchItems) do + if #items >= self.maxFetchPerSearch then + break + end + if not isValueInTable(fetchedItemIds, value.id) then + table.insert(items, value) + table.insert(fetchedItemIds, value.id) + end + end + local fillCount = self.maxFetchPerSearch - #items + if fillCount > 0 then + -- fill with previous search results + local unfetchedItemIds = {} + for _, value in pairs(previousSearchItemIds) do + if #unfetchedItemIds >= fillCount then + break + end + if not isValueInTable(fetchedItemIds, value) then + table.insert(unfetchedItemIds, value) + end + end + self:FetchResults(unfetchedItemIds, previousSearchId, function(newItems, errMsg) + if errMsg then + return callback(nil, errMsg) + end + items = tableConcat(items, newItems) + callback(items, errMsg) + end) + else + callback(items, errMsg) + end + end) + else + -- Search not clipped and result count satisfy maxFetchPerSearch, proceed normally + self:FetchResults(response.result, response.id, callback) + end + else + if response.total < self.maxFetchPerSearch then -- Less than maximum items retrieved lower weight to try and get more. + local queryJson = dkjson.decode(query) + queryJson.query.stats[1].value.min = queryJson.query.stats[1].value.min / 2 + query = dkjson.encode(queryJson) + self:PerformSearch(realm, league, query, performSearchCallback) + else -- Search clipped, fetch highest weight item, update query weight and repeat search + previousSearchItemIds = response.result + previousSearchId = response.id + local firstResultBatch = {unpack(response.result, 1, math.min(#response.result, 10))} + self:FetchResults(firstResultBatch, response.id, function(items, errMsg) + if errMsg then + return callback(nil, errMsg) + end + previousSearchItems = items + local highestWeight = items[1].weight + local queryJson = dkjson.decode(query) + queryJson.query.stats[1].value.min = (tonumber(highestWeight) + queryJson.query.stats[1].value.min) / 2 + query = dkjson.encode(queryJson) + self:PerformSearch(realm, league, query, performSearchCallback) + end) + end + end + end + self:PerformSearch(realm, league, query, performSearchCallback) +end + +---Perform search and run callback function on returned item hashes. +---Item info has to be fetched separately +---@param league string +---@param query string +---@param callback fun(response:table, errMsg:string) +function TradeQueryRequestsClass:PerformSearch(realm, league, query, callback) + table.insert(self.requestQueue["search"], { + url = self:buildUrl(self.hostName .. "api/trade2/search", realm, league), + body = query, + callback = function(response, errMsg) + if errMsg and not errMsg:find("Response code: 400") then + return callback(nil, errMsg) + end + local response = dkjson.decode(response) + if not response then + errMsg = "Failed to Get Trade response" + return callback(nil, errMsg) + end + if not response.result or #response.result == 0 then + if response.error then + if not (response.error.code and response.error.message) then + errMsg = "Encountered unknown error, check console for details." + ConPrintf("Unknown error: %s", utils.stringify(response.error)) + callback(response, errMsg) + end + if response.error.message:find("Logging in will increase this limit") then + errMsg = "Authorization is invalid. Please Re-Log and reset" + else + -- Report unhandled error + errMsg = "[ " .. response.error.code .. ": " .. response.error.message .. " ]" + end + else + ConPrintf("Found 0 results for %sapi/trade2/search/%s/%s", self.hostName, league, response.id) + errMsg = "No Matching Results Found" + end + return callback(response, errMsg) + end + callback(response, errMsg) + end, + }) +end + +---Fetch item details for itemHashes +---@param itemHashes string[] +---@param queryId string +---@param callback fun(items:table, errMsg:string) +function TradeQueryRequestsClass:FetchResults(itemHashes, queryId, callback) + local quantity_found = math.min(#itemHashes, self.maxFetchPerSearch) + local max_block_size = 10 + local items = {} + for fetch_block_start = 1, quantity_found, max_block_size do + local fetch_block_end = math.min(fetch_block_start + max_block_size - 1, quantity_found) + local param_item_hashes = table.concat({unpack(itemHashes, fetch_block_start, fetch_block_end)}, ",") + local fetch_url = self.hostName .. "api/trade2/fetch/"..param_item_hashes.."?query="..queryId + self:FetchResultBlock(fetch_url, function(itemBlock, errMsg) + if errMsg then + return callback(nil, errMsg) + end + for _, item in pairs(itemBlock) do + table.insert(items, item) + end + -- finished fetching item blocks + if #items >= quantity_found then + callback(items) + end + end) + end +end + +---Fetch details for paginated items +---@param url string +---@param callback fun(items: table, errMsg:string) +function TradeQueryRequestsClass:FetchResultBlock(url, callback) + table.insert(self.requestQueue["fetch"], { + url = url, + callback = function(response, errMsg) + if errMsg then + return callback(nil, errMsg) + end + local response, response_err = dkjson.decode(response) + if not response or not response.result then + if response_err then + errMsg = "JSON Parse Error: " .. (errMsg or "") + else + errMsg = "Failed to Get Trade Items: " .. (errMsg or "") + end + return callback(nil, errMsg) + end + local items = {} + for _, trade_entry in pairs(response.result) do + local item = trade_entry.item + local spirit + local armour + local evasion + local es + local charmSlots + local quality + local radius + local limit + local t_insert = table.insert + -- local catalystList = {"Abrasive", "Accelerating", "Fertile", "Imbued", "Intrinsic", "Noxious", "Prismatic", "Tempering", "Turbulent", "Unstable"} + + if item.properties then + for _, property in ipairs(item.properties) do + local name = escapeGGGString(property.name) + if name == "Armour" then + armour = property.values[1][1] + elseif name == "Evasion Rating" then + evasion = property.values[1][1] + elseif name == "Energy Shield" then + es = property.values[1][1] + elseif name == "Quality" then + quality = property.values[1][1]:sub(2, -2) -- remove + and % on quality value + elseif name == "Spirit" then + spirit = property.values[1][1] + elseif name == "Charm Slots" then + charmSlots = property.values[1][1] + -- elseif name == "Quality (Mana Modifiers)" then + -- catalyst quality stuff tbd it all needs reworking anyway as it has changed. + elseif name == "Radius" then + radius = property.values[1][1] + elseif name == "Limited to" then + limit = property.values[1][1] + end + end + end + + local rawLines = { } + t_insert(rawLines, "Rarity: " .. item.rarity) + -- item.name is empty when magic and full magic name is in typeLine but typeLine == baseType when rare. + if item.name ~= "" then + t_insert(rawLines, item.name) + end + t_insert(rawLines, item.typeLine) + + if charmSlots then + t_insert(rawLines, "Charm Slots: " .. charmSlots) + end + + if spirit then + t_insert(rawLines, "Spirit: " .. spirit) + end + + if armour then + t_insert(rawLines, "Armour: " .. armour) + end + if evasion then + t_insert(rawLines, "Evasion: " ..evasion) + end + if es then + t_insert(rawLines, "Energy Shield: " .. es) + end + + -- if self.catalyst and self.catalyst > 0 then + -- t_insert(rawLines, "Catalyst: " .. catalystList[self.catalyst]) + -- end + -- if self.catalystQuality then + -- t_insert(rawLines, "CatalystQuality: " .. self.catalystQuality) + -- end + + if item.ilvl then + t_insert(rawLines, "Item Level: " .. item.ilvl) + end + if quality then + t_insert(rawLines, "Quality: " .. quality) + end + if item.sockets then + local socketString = "" + for _, _ in ipairs(item.sockets) do + socketString = socketString .. "S " + end + socketString = socketString:gsub(" $", "") + t_insert(rawLines, "Sockets: " .. socketString) + end + + if item.requirements then + for _, requirement in ipairs(item.requirements) do + if requirement.name == "Level" then + t_insert(rawLines, "LevelReq: " .. requirement.values[1][1]) + end + end + end + + if radius then + t_insert(rawLines, "Radius: " .. radius) + end + if limit then + t_insert(rawLines, "Limited to: " .. limit) + end + + -- ensure these fields are initialised + item.enchantMods = item.enchantMods or { } + item.fracturedMods = item.fracturedMods or { } + item.desecratedMods = item.desecratedMods or { } + item.craftedMods = item.craftedMods or { } + item.runeMods = item.runeMods or { } + item.implicitMods = item.implicitMods or { } + item.explicitMods = item.explicitMods or { } + + local function processLine(modLine) + local s = "" + for flagName, flag in pairs(modLine.flags or {}) do + if flag then + s = s .. string.format("{%s}", flagName) + end + end + return s .. escapeGGGString(modLine.description) + end + t_insert(rawLines, "Implicits: " .. (#item.enchantMods + #item.runeMods + #item.implicitMods)) + for _, modLine in ipairs(item.enchantMods or {}) do + t_insert(rawLines, "{enchant}" .. processLine(modLine)) + end + for _, modLine in ipairs(item.runeMods or {}) do + t_insert(rawLines, "{enchant}{rune}" .. processLine(modLine)) + end + for _, modLine in ipairs(item.implicitMods or {}) do + t_insert(rawLines, processLine(modLine)) + end + for _, modLine in ipairs(item.explicitMods or {}) do + t_insert(rawLines, processLine(modLine)) + end + if item.mirrored then + t_insert(rawLines, "Mirrored") + end + if item.doubleCorrupted then + t_insert(rawLines, "Twice Corrupted") + elseif item.corrupted then + t_insert(rawLines, "Corrupted") + end + if item.sanctified then + t_insert(rawLines, "Sanctified") + end + + local pseudoMod = trade_entry.item.pseudoMods and trade_entry.item.pseudoMods[1] + local pseudoModLine = pseudoMod and (pseudoMod.description or pseudoMod) + table.insert(items, { + amount = trade_entry.listing.price.amount, + currency = trade_entry.listing.price.currency, + priceType = trade_entry.listing.price.type, + item_string = table.concat(rawLines, "\n"), + whisper = trade_entry.listing.whisper, + trader = trade_entry.listing.account.name, + weight = trade_entry.item.pseudoMods and pseudoModLine:match("Sum: (.+)") or "0", + id = trade_entry.id + }) + end + return callback(items) + end + }) +end + +---@param callback fun(items:table, errMsg:string, query: string?) +function TradeQueryRequestsClass:SearchWithURL(url, callback) + local prefix = self.hostName .. "trade2/search/" + if url:sub(1, #prefix) ~= prefix then + return callback(nil, "Invalid URL", nil) + end + local subpath = url:sub(#prefix + 1) + local paths = {} + for path in subpath:gmatch("[^/]+") do + table.insert(paths, path) + end + if #paths < 2 or #paths > 3 then + return callback(nil, "Invalid URL", nil) + end + local realm, league, queryId + if #paths == 3 then + realm = paths[1] + end + -- URL path segments are already escaped; buildUrl encodes the league again. + league = paths[#paths-1]:gsub("%%(%x%x)", function(hex) + return string.char(tonumber(hex, 16)) + end) + queryId = paths[#paths] + self:FetchSearchQuery(realm, league, queryId, function(query, errMsg) + if errMsg then + return callback(nil, errMsg, nil) + end + + -- update sorting on provided url to sort by weights. + local json_data = dkjson.decode(query) + if type(json_data) ~= "table" or json_data.error or type(json_data.query) ~= "table" then + return callback(nil, type(json_data) == "table" and json_data.error or "Failed to parse search query JSON", nil) + end + if json_data.query.stats and json_data.query.stats[1] and json_data.query.stats[1].type == "weight" then + json_data.sort = {} + json_data.sort["statgroup.0"] = "desc" + else + json_data.sort = { price = "asc"} + end + query = dkjson.encode(json_data) + + self:SearchWithQuery(realm, league, query, function(items, searchErrMsg) + callback(items, searchErrMsg, query) + end) + end) +end + +---Fetch query data needed to perform the search +---@param queryId string +---@param league string +---@param callback fun(query:string, errMsg:string) +function TradeQueryRequestsClass:FetchSearchQuery(realm, league, queryId, callback) + -- Browser share links can contain a gzip-compressed query instead of a saved ID. + if queryId:sub(1, 4) == "H4sI" then + local ok, query = pcall(function() + local compressed = require("base64").decode(queryId:gsub("-", "+"):gsub("_", "/")) + return LoadModule("Modules/TradeQueryDecode")(compressed) + end) + if not ok or not query then + return callback(nil, "Failed to decode compressed search query") + end + local data = dkjson.decode(query) + if type(data) ~= "table" then + return callback(nil, "Failed to parse compressed search query") + end + return callback(dkjson.encode({ query = data })) + end + local url = self:buildUrl(self.hostName .. "api/trade2/search", realm, league, queryId) + table.insert(self.requestQueue["search"], { + url = url, + callback = function(response, errMsg) + if errMsg then + return callback(nil, errMsg) + end + local json_data = dkjson.decode(response) + if not json_data or json_data.error then + errMsg = json_data and json_data.error or "Failed to get search query" + end + callback(response, errMsg) + end + }) +end + +--- Fetches the list of all available leagues using trade2 league API +---@param realm string +---@param callback fun(query:table, errMsg:string) +function TradeQueryRequestsClass:FetchLeagues(realm, callback) + local header = "Authorization: Bearer ".. (main.api.authToken or "") + launch:DownloadPage( + self.hostName .. "api/trade2/data/leagues", + function(response, errMsg) + if errMsg then + return callback({"Standard", "Hardcore"}, errMsg) + end + local json_data = dkjson.decode(response.body) + if not json_data or json_data.error then + errMsg = json_data and json_data.error or "Failed to parse trade leagues JSON" + end + local leagues = {} + for _, value in pairs(json_data.result) do + if value.realm == realm then + table.insert(leagues, value.id) + end + end + callback(leagues, errMsg) + end, + {header = header} + ) +end + +--- Build search and trade URLs with proper encoding +---@param root string +---@param realm string +---@param league string +---@param queryId string +function TradeQueryRequestsClass:buildUrl(root, realm, league, queryId) + local result = root + if realm and realm ~='pc' then + result = result .. "/" .. realm + end + local encodedLeague = league:gsub("[^%w%-%.%_%~]", function(c) + return string.format("%%%02X", string.byte(c)) + end):gsub(" ", "+") + result = result .. "/" .. encodedLeague + if queryId then + result = result .. "/" .. queryId + end + return result +end