From f4c38475386f18551962cf96351b2a5e926c179c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=9B=90=EB=8D=94?= Date: Mon, 11 Jan 2021 15:33:33 +0900 Subject: [PATCH] feat: added tlru caching close: https://github.com/koreanbots/v2-testing/issues/22 --- pages/bots/[id].tsx | 12 +-- pages/index.tsx | 8 +- pages/users/[id].tsx | 4 +- utils/Fetch.ts | 231 ------------------------------------------- utils/Query.ts | 171 ++++++++++++++++++++++++++++++-- utils/index.ts | 3 +- 6 files changed, 174 insertions(+), 255 deletions(-) delete mode 100644 utils/Fetch.ts diff --git a/pages/bots/[id].tsx b/pages/bots/[id].tsx index 27cdb15..5f5f2d6 100644 --- a/pages/bots/[id].tsx +++ b/pages/bots/[id].tsx @@ -14,13 +14,13 @@ import NotFound from '../404' import SEO from '../../components/SEO' import LongButton from '../../components/LongButton' import { git, Status } from '../../utils/Constants' -import { Fetch } from '../../utils' +import { Query } from '../../utils' import { formatNumber } from '../../utils/Tools' import Advertisement from '../../components/Advertisement' import Link from 'next/link' const Bots: NextPage = ({ data, date }) => { - if (!data.id) return + if (!data || !data.id) return return ( = ({ data, date }) => { } export const getServerSideProps = async (ctx: Context) => { - const data = await Fetch.bot.load(ctx.query.id) + const data = await Query.get.bot.load(ctx.query.id) ?? {} return { props: { data, - date: SnowflakeUtil.deconstruct(data.id ?? '0').date.toJSON() + date: SnowflakeUtil.deconstruct(data?.id ?? '0').date.toJSON() }, } } @@ -225,9 +225,9 @@ interface BotsProps { votes: string } interface Context extends NextPageContext { - query: Query + query: URLQuery } -interface Query extends ParsedUrlQuery { +interface URLQuery extends ParsedUrlQuery { id: string } diff --git a/pages/index.tsx b/pages/index.tsx index 240d700..c2f3dfa 100644 --- a/pages/index.tsx +++ b/pages/index.tsx @@ -5,7 +5,7 @@ import Container from '../components/Container' import LongButton from '../components/LongButton' import Wave from '../components/Wave' import { BotList } from '../types' -import { Fetch } from '../utils' +import { Query } from '../utils' const Index: NextPage = ({ votes, newBots, trusted }) => { return ( @@ -61,9 +61,9 @@ const Index: NextPage = ({ votes, newBots, trusted }) => { } export const getServerSideProps = async() => { - const votes = await Fetch.botListVotes.load(1) - const newBots = await Fetch.botListNew.load(1) - const trusted = await Fetch.botListTrusted.load(1) + const votes = await Query.get.list.votes.load(1) + const newBots = await Query.get.list.new.load(1) + const trusted = await Query.get.list.trusted.load(1) return { props: { votes, newBots, trusted }} diff --git a/pages/users/[id].tsx b/pages/users/[id].tsx index 1ec6f18..6608efd 100644 --- a/pages/users/[id].tsx +++ b/pages/users/[id].tsx @@ -1,6 +1,6 @@ import { NextPage, NextPageContext } from 'next' import { SnowflakeUtil } from 'discord.js' -import { Fetch } from '../../utils' +import { Query } from '../../utils' import { ParsedUrlQuery } from 'querystring' import { josa } from 'josa' import { Bot, User } from '../../types' @@ -88,7 +88,7 @@ const Users: NextPage = ({ data }) => { } export const getServerSideProps = async (ctx: Context) => { - const data = await Fetch.user.load(ctx.query.id) + const data = await Query.get.user.load(ctx.query.id) return { props: { data, date: SnowflakeUtil.deconstruct(data.id ?? '0')?.date?.toJSON() } } } diff --git a/utils/Fetch.ts b/utils/Fetch.ts deleted file mode 100644 index 17099ef..0000000 --- a/utils/Fetch.ts +++ /dev/null @@ -1,231 +0,0 @@ -import DataLoader from 'dataloader' -import fs from 'fs' -import { Bot, ListType, User, BotList } from '../types' -import knex from './Knex' -import { cats } from './Constants' -import { botListArgument } from './Yup' -import { ReactText } from 'react' - -const publicPem = fs.readFileSync('./public.pem') -const privateKey = fs.readFileSync('./private.key') - -const bot = new DataLoader( - async (ids: string[]) => - (await Promise.all(ids.map(async (el: string) => getBot(el)))).map(row => ({ ...row })) -) - -new DataLoader(async ()=> { return []}) -const botWithNoUser = new DataLoader( - async (ids: string[]) => - (await Promise.all(ids.map(async (el: string) => getBot(el, false)))).map(row => ({ ...row })) -) - -const user = new DataLoader( - async (ids: string[]) => - (await Promise.all(ids.map((el: string) => getUser(el)))).map(row => ({ ...row })) -) - -const userWithNoBot = new DataLoader( - async (ids: string[]) => - (await Promise.all(ids.map((el: string) => getUser(el, false)))).map(row => ({ ...row })) -) - -const botListVotes = new DataLoader( - async (page: number[]) => - (await Promise.all(page.map((el) => getBotList('VOTE', el)))).map(row => ({ ...row })), - { - batchScheduleFn: callback => setTimeout(callback, 100), - } -) - -const botListNew = new DataLoader( - async (page: number[]) => - (await Promise.all(page.map((el) => getBotList('NEW', el)))).map(row => ({ ...row })), - { - batchScheduleFn: callback => setTimeout(callback, 100), - } -) - -const botListTrusted = new DataLoader( - async (page: number[]) => - (await Promise.all(page.map((el) => getBotList('TRUSTED', el)))).map(row => ({ ...row })), - { - batchScheduleFn: callback => setTimeout(callback, 100), - } -) - -async function getBot(id: string, owners = true): Promise { - const res = await knex('bots') - .select([ - 'id', - 'owners', - 'lib', - 'prefix', - 'votes', - 'servers', - 'intro', - 'desc', - 'web', - 'git', - 'url', - 'category', - 'status', - 'name', - 'avatar', - 'tag', - 'verified', - 'trusted', - 'partnered', - 'discord', - 'boosted', - 'state', - 'vanity', - 'bg', - 'banner', - ]) - .where({ id }) - .orWhere({ vanity: id, trusted: true }) - .orWhere({ vanity: id, partnered: true }) - if (res[0]) { - res[0].category = JSON.parse(res[0].category) - res[0].owners = JSON.parse(res[0].owners) - if (owners) - res[0].owners = await Promise.all( - res[0].owners.map(async (u: string) => await userWithNoBot.load(u)) - ) - res[0].owners = res[0].owners.filter((el: User | null) => el).map((row: User) => ({ ...row })) - res[0].vanity = ((res[0].trusted || res[0].partnered) && res[0].vanity) ?? null - } - - return res[0] || null -} - -async function getUser(id: string, bots = true): Promise { - const res = await knex('users') - .select(['id', 'avatar', 'tag', 'username', 'perm', 'github']) - .where({ id }) - if (res[0]) { - const owned = await knex('bots') - .select(['id']) - .where('owners', 'like', `%${id}%`) - if (bots) res[0].bots = await Promise.all(owned.map(async b => await botWithNoUser.load(b.id))) - else res[0].bots = owned.map(async b => b.id) - res[0].bots = res[0].bots.filter((el: Bot | null) => el).map((row: User) => ({ ...row })) - } - - return res[0] || null -} - -async function getBotList(type: ListType, page = 1, query?: string):Promise { - let res: { id: string }[] - let count: ReactText - if (type === 'VOTE') { - count = (await knex('bots').count())[0]['count(*)'] - res = await knex('bots') - .orderBy('votes', 'desc') - .orderBy('servers', 'desc') - .limit(16) - .offset(((page ? Number(page) : 1) - 1) * 16) - .select(['id']) - } else if (type === 'TRUSTED') { - count = ( - await knex('bots') - .where({ trusted: true }) - .count() - )[0]['count(*)'] - res = await knex('bots') - .where({ trusted: true }) - .orderByRaw('RAND()') - .limit(16) - .offset(((page ? Number(page) : 1) - 1) * 16) - .select(['id']) - } else if (type === 'NEW') { - count = ( - await knex('bots') - .count() - )[0]['count(*)'] - res = await knex('bots') - .orderBy('date', 'desc') - .limit(16) - .offset(((page ? Number(page) : 1) - 1) * 16) - .select(['id']) - } else if (type === 'PARTNERED') { - count = ( - await knex('bots') - .where({ partnered: true }) - .count() - )[0]['count(*)'] - res = await knex('bots') - .where({ partnered: true }) - .orderByRaw('RAND()') - .limit(16) - .offset(((page ? Number(page) : 1) - 1) * 16) - .select(['id']) - } else if (type === 'CATEGORY') { - if (!query) throw new Error('쿼리가 누락되었습니다.') - if (!cats.includes(query)) throw new Error('알 수 없는 카테고리입니다.') - count = ( - await knex('bots') - .where('category', 'like', `%${decodeURI(query)}%`) - .count() - )[0]['count(*)'] - res = await knex('bots') - .where('category', 'like', `%${decodeURI(query)}%`) - .orderBy('votes', 'desc') - .orderBy('servers', 'desc') - .limit(16) - .offset(((page ? Number(page) : 1) - 1) * 16) - .select(['id']) - } else if (type === 'SEARCH') { - if (!query) throw new Error('쿼리가 누락되었습니다.') - try { - new RegExp(query) - count = ( - await knex('bots') - .where('name', 'REGEXP', query) - .orWhere('intro', 'REGEXP', query) - .orWhere('desc', 'REGEXP', query) - .orderBy('votes', 'desc') - .orderBy('servers', 'desc') - .count() - )[0]['count(*)'] - - res = await knex('bots') - .where('name', 'REGEXP', query) - .orWhere('intro', 'REGEXP', query) - .orWhere('desc', 'REGEXP', query) - .orderBy('votes', 'desc') - .orderBy('servers', 'desc') - .limit(16) - .offset(((page ? Number(page) : 1) - 1) * 16) - .select(['id']) - } catch (e) { - count = ( - await knex('bots') - .where('name', 'LIKE', `%${query}%`) - .orWhere('intro', 'LIKE', `%${query}%`) - .orWhere('desc', 'LIKE', `%${query}%`) - .orderBy('votes', 'desc') - .orderBy('servers', 'desc') - .count() - )[0]['count(*)'] - - res = await knex('bots') - .where('name', 'LIKE', `%${query}%`) - .orWhere('intro', 'LIKE', `%${query}%`) - .orWhere('desc', 'LIKE', `%${query}%`) - .orderBy('votes', 'desc') - .orderBy('servers', 'desc') - .limit(16) - .offset(((page ? Number(page) : 1) - 1) * 16) - .select(['id']) - } - } else { - count = 1 - res = [] - } - - return { type, data: (await Promise.all(res.map(async el => await getBot(el.id)))).map(r=> ({...r})), currentPage: page, totalPage: Math.ceil(Number(count) / 16) } -} - -export default { bot, user, botListVotes, botListNew, botListTrusted } diff --git a/utils/Query.ts b/utils/Query.ts index 825fef4..d69c5fc 100644 --- a/utils/Query.ts +++ b/utils/Query.ts @@ -1,15 +1,18 @@ import fetch from 'node-fetch' import jwt from 'jsonwebtoken' import fs from 'fs' +import { TLRU } from 'tlru' -import { Bot, User } from '../types' +import { Bot, User, ListType, BotList } from '../types' +import { cats } from './Constants' import knex from './Knex' +import DataLoader from 'dataloader' const publicPem = fs.readFileSync('./public.pem') const privateKey = fs.readFileSync('./private.key') -export async function getBot(id: string, owners = true): Promise { +async function getBot(id: string, owners=true) { const res = await knex('bots') .select([ 'id', @@ -39,20 +42,23 @@ export async function getBot(id: string, owners = true): Promise { 'banner', ]) .where({ id }) - .orWhere({ vanity: id, boosted: 1 }) + .orWhere({ vanity: id, trusted: true }) + .orWhere({ vanity: id, partnered: true }) if (res[0]) { res[0].category = JSON.parse(res[0].category) res[0].owners = JSON.parse(res[0].owners) - if (owners) - res[0].owners = await Promise.all(res[0].owners.map(async (u: string) => await getUser(u))) - res[0].owners = res[0].owners.filter((el: User | null) => el).map((row: User) => ({ ...row })) - res[0].vanity = res[0].vanity && (res[0].boosted || res[0].trusted || res[0].partnered) + res[0].vanity = ((res[0].trusted || res[0].partnered) && res[0].vanity) ?? null } + if (owners) + res[0].owners = await Promise.all( + res[0].owners.map(async (u: string) => await get._rawUser.load(u)) + ) + res[0].owners = res[0].owners.filter((el: User | null) => el).map((row: User) => ({ ...row })) - return res[0] || null + return res[0] ?? null } -export async function getUser(id: string, bots = true): Promise { +async function getUser(id: string, bots = true) { const res = await knex('users') .select(['id', 'avatar', 'tag', 'username', 'perm', 'github']) .where({ id }) @@ -60,10 +66,155 @@ export async function getUser(id: string, bots = true): Promise { const owned = await knex('bots') .select(['id']) .where('owners', 'like', `%${id}%`) - if (bots) res[0].bots = await Promise.all(owned.map(async b => await getBot(b.id, false))) + if (bots) res[0].bots = await Promise.all(owned.map(async b => await get._rawBot.load(b.id))) else res[0].bots = owned.map(async b => b.id) res[0].bots = res[0].bots.filter((el: Bot | null) => el).map((row: User) => ({ ...row })) } return res[0] || null } + +async function getBotList(type: ListType, page = 1, query?: string):Promise { + let res: { id: string }[] + let count:string|number + if (type === 'VOTE') { + count = (await knex('bots').count())[0]['count(*)'] + res = await knex('bots') + .orderBy('votes', 'desc') + .orderBy('servers', 'desc') + .limit(16) + .offset(((page ? Number(page) : 1) - 1) * 16) + .select(['id']) + } else if (type === 'TRUSTED') { + count = ( + await knex('bots') + .where({ trusted: true }) + .count() + )[0]['count(*)'] + res = await knex('bots') + .where({ trusted: true }) + .orderByRaw('RAND()') + .limit(16) + .offset(((page ? Number(page) : 1) - 1) * 16) + .select(['id']) + } else if (type === 'NEW') { + count = ( + await knex('bots') + .count() + )[0]['count(*)'] + res = await knex('bots') + .orderBy('date', 'desc') + .limit(16) + .offset(((page ? Number(page) : 1) - 1) * 16) + .select(['id']) + } else if (type === 'PARTNERED') { + count = ( + await knex('bots') + .where({ partnered: true }) + .count() + )[0]['count(*)'] + res = await knex('bots') + .where({ partnered: true }) + .orderByRaw('RAND()') + .limit(16) + .offset(((page ? Number(page) : 1) - 1) * 16) + .select(['id']) + } else if (type === 'CATEGORY') { + if (!query) throw new Error('쿼리가 누락되었습니다.') + if (!cats.includes(query)) throw new Error('알 수 없는 카테고리입니다.') + count = ( + await knex('bots') + .where('category', 'like', `%${decodeURI(query)}%`) + .count() + )[0]['count(*)'] + res = await knex('bots') + .where('category', 'like', `%${decodeURI(query)}%`) + .orderBy('votes', 'desc') + .orderBy('servers', 'desc') + .limit(16) + .offset(((page ? Number(page) : 1) - 1) * 16) + .select(['id']) + } else if (type === 'SEARCH') { + if (!query) throw new Error('쿼리가 누락되었습니다.') + try { + new RegExp(query) + count = ( + await knex('bots') + .where('name', 'REGEXP', query) + .orWhere('intro', 'REGEXP', query) + .orWhere('desc', 'REGEXP', query) + .orderBy('votes', 'desc') + .orderBy('servers', 'desc') + .count() + )[0]['count(*)'] + + res = await knex('bots') + .where('name', 'REGEXP', query) + .orWhere('intro', 'REGEXP', query) + .orWhere('desc', 'REGEXP', query) + .orderBy('votes', 'desc') + .orderBy('servers', 'desc') + .limit(16) + .offset(((page ? Number(page) : 1) - 1) * 16) + .select(['id']) + } catch (e) { + count = ( + await knex('bots') + .where('name', 'LIKE', `%${query}%`) + .orWhere('intro', 'LIKE', `%${query}%`) + .orWhere('desc', 'LIKE', `%${query}%`) + .orderBy('votes', 'desc') + .orderBy('servers', 'desc') + .count() + )[0]['count(*)'] + + res = await knex('bots') + .where('name', 'LIKE', `%${query}%`) + .orWhere('intro', 'LIKE', `%${query}%`) + .orWhere('desc', 'LIKE', `%${query}%`) + .orderBy('votes', 'desc') + .orderBy('servers', 'desc') + .limit(16) + .offset(((page ? Number(page) : 1) - 1) * 16) + .select(['id']) + } + } else { + count = 1 + res = [] + } + + return { type, data: (await Promise.all(res.map(async el => await getBot(el.id)))).map(r=> ({...r})), currentPage: page, totalPage: Math.ceil(Number(count) / 16) } +} + +export const get = { + bot: new DataLoader( + async (ids: string[]) => + (await Promise.all(ids.map(async (el: string) => await getBot(el)))).map(row => ({ ...row })) + , { cacheMap: new TLRU({ maxStoreSize: 50, maxAgeMs: 60000 }) }), + _rawBot: new DataLoader( + async (ids: string[]) => + (await Promise.all(ids.map(async (el: string) => await getBot(el, false)))).map(row => ({ ...row })) + , { cacheMap: new TLRU({ maxStoreSize: 50, maxAgeMs: 60000 }) }), + user: new DataLoader( + async (ids: string[]) => + (await Promise.all(ids.map(async (el: string) => await getUser(el)))).map(row => ({ ...row })) + , { cacheMap: new TLRU({ maxStoreSize: 50, maxAgeMs: 60000 }) }), + _rawUser: new DataLoader( + async (ids: string[]) => + (await Promise.all(ids.map(async (el: string) => await getUser(el, false)))).map(row => ({ ...row })) + , { cacheMap: new TLRU({ maxStoreSize: 50, maxAgeMs: 60000 }) }), + list: { + votes: new DataLoader( + async (pages: number[]) => + (await Promise.all(pages.map(async (page: number) => await getBotList('VOTE', page)))).map(row => ({ ...row })) + , { cacheMap: new TLRU({ maxStoreSize: 50, maxAgeMs: 600000 }) }), + new: new DataLoader( + async (pages: number[]) => + (await Promise.all(pages.map(async (page: number) => await getBotList('NEW', page)))).map(row => ({ ...row })) + , { cacheMap: new TLRU({ maxStoreSize: 50, maxAgeMs: 1800000 }) }), + trusted: new DataLoader( + async (pages: number[]) => + (await Promise.all(pages.map(async (page: number) => await getBotList('TRUSTED', page)))).map(row => ({ ...row })) + , { cacheMap: new TLRU({ maxStoreSize: 50, maxAgeMs: 3600000 }) }), + } +} \ No newline at end of file diff --git a/utils/index.ts b/utils/index.ts index 6e0173a..75ea01b 100644 --- a/utils/index.ts +++ b/utils/index.ts @@ -1,6 +1,5 @@ import * as Query from './Query' import { } from './Tools' import ResponseWrapper from './ResponseWrapper' -import Fetch from './Fetch' -export { Query, Fetch, ResponseWrapper } +export { Query, ResponseWrapper }