feat: added tlru caching

close: https://github.com/koreanbots/v2-testing/issues/22

pages/bots/[id].tsx

@@ -14,13 +14,13 @@ import NotFound from '../404'
 import SEO from '../../components/SEO'
 import LongButton from '../../components/LongButton'
 import { git, Status } from '../../utils/Constants'
-import { Fetch } from '../../utils'
+import { Query } from '../../utils'
 import { formatNumber } from '../../utils/Tools'
 import Advertisement from '../../components/Advertisement'
 import Link from 'next/link'
 
 const Bots: NextPage<BotsProps> = ({ data, date }) => {
-	if (!data.id) return <NotFound />
+	if (!data || !data.id) return <NotFound />
 	return (
 		<Container paddingTop className='py-10'>
 			<SEO
@@ -208,11 +208,11 @@ const Bots: NextPage<BotsProps> = ({ data, date }) => {
 }
 
 export const getServerSideProps = async (ctx: Context) => {
-	const data = await Fetch.bot.load(ctx.query.id)
+	const data = await Query.get.bot.load(ctx.query.id) ?? {}
 	return {
 		props: {
 			data,
-			date: SnowflakeUtil.deconstruct(data.id ?? '0').date.toJSON()
+			date: SnowflakeUtil.deconstruct(data?.id ?? '0').date.toJSON()
 		},
 	}
 }
@@ -225,9 +225,9 @@ interface BotsProps {
 	votes: string
 }
 interface Context extends NextPageContext {
-	query: Query
+	query: URLQuery
 }
 
-interface Query extends ParsedUrlQuery {
+interface URLQuery extends ParsedUrlQuery {
 	id: string
 }

pages/index.tsx

@@ -5,7 +5,7 @@ import Container from '../components/Container'
 import LongButton from '../components/LongButton'
 import Wave from '../components/Wave'
 import { BotList } from '../types'
-import { Fetch } from '../utils'
+import { Query } from '../utils'
 
 const Index: NextPage<IndexProps> = ({ votes, newBots, trusted }) => {
 	return (
@@ -61,9 +61,9 @@ const Index: NextPage<IndexProps> = ({ votes, newBots, trusted }) => {
 }
 
 export const getServerSideProps = async() => {
-	const votes = await Fetch.botListVotes.load(1)
+	const votes = await Query.get.list.votes.load(1)
-	const newBots = await Fetch.botListNew.load(1)
+	const newBots = await Query.get.list.new.load(1)
-	const trusted = await Fetch.botListTrusted.load(1)
+	const trusted = await Query.get.list.trusted.load(1)
 
 	return { props: { votes, newBots, trusted }}
 

pages/users/[id].tsx

@@ -1,6 +1,6 @@
 import { NextPage, NextPageContext } from 'next'
 import { SnowflakeUtil } from 'discord.js'
-import { Fetch } from '../../utils'
+import { Query } from '../../utils'
 import { ParsedUrlQuery } from 'querystring'
 import { josa } from 'josa'
 import { Bot, User } from '../../types'
@@ -88,7 +88,7 @@ const Users: NextPage<UserProps> = ({ data }) => {
 }
 
 export const getServerSideProps = async (ctx: Context) => {
-	const data = await Fetch.user.load(ctx.query.id)
+	const data = await Query.get.user.load(ctx.query.id)
 	return { props: { data, date: SnowflakeUtil.deconstruct(data.id ?? '0')?.date?.toJSON() } }
 }
 

utils/Fetch.ts

@@ -1,231 +0,0 @@
-import DataLoader from 'dataloader'
-import fs from 'fs'
-import { Bot, ListType, User, BotList } from '../types'
-import knex from './Knex'
-import { cats } from './Constants'
-import { botListArgument } from './Yup'
-import { ReactText } from 'react'
-
-const publicPem = fs.readFileSync('./public.pem')
-const privateKey = fs.readFileSync('./private.key')
-
-const bot = new DataLoader(
-	async (ids: string[]) =>
-		(await Promise.all(ids.map(async (el: string) => getBot(el)))).map(row => ({ ...row }))
-)
-
-new DataLoader(async ()=> { return []})
-const botWithNoUser = new DataLoader(
-	async (ids: string[]) =>
-		(await Promise.all(ids.map(async (el: string) => getBot(el, false)))).map(row => ({ ...row }))
-)
-
-const user = new DataLoader(
-	async (ids: string[]) =>
-		(await Promise.all(ids.map((el: string) => getUser(el)))).map(row => ({ ...row }))
-)
-
-const userWithNoBot = new DataLoader(
-	async (ids: string[]) =>
-		(await Promise.all(ids.map((el: string) => getUser(el, false)))).map(row => ({ ...row }))
-)
-
-const botListVotes = new DataLoader(
-	async (page: number[]) =>
-		(await Promise.all(page.map((el) => getBotList('VOTE', el)))).map(row => ({ ...row })),
-	{
-		batchScheduleFn: callback => setTimeout(callback, 100),
-	}
-)
-
-const botListNew = new DataLoader(
-	async (page: number[]) =>
-		(await Promise.all(page.map((el) => getBotList('NEW', el)))).map(row => ({ ...row })),
-	{
-		batchScheduleFn: callback => setTimeout(callback, 100),
-	}
-)
-
-const botListTrusted = new DataLoader(
-	async (page: number[]) =>
-		(await Promise.all(page.map((el) => getBotList('TRUSTED', el)))).map(row => ({ ...row })),
-	{
-		batchScheduleFn: callback => setTimeout(callback, 100),
-	}
-)
-
-async function getBot(id: string, owners = true): Promise<Bot> {
-	const res = await knex('bots')
-		.select([
-			'id',
-			'owners',
-			'lib',
-			'prefix',
-			'votes',
-			'servers',
-			'intro',
-			'desc',
-			'web',
-			'git',
-			'url',
-			'category',
-			'status',
-			'name',
-			'avatar',
-			'tag',
-			'verified',
-			'trusted',
-			'partnered',
-			'discord',
-			'boosted',
-			'state',
-			'vanity',
-			'bg',
-			'banner',
-		])
-		.where({ id })
-		.orWhere({ vanity: id, trusted: true })
-		.orWhere({ vanity: id, partnered: true })
-	if (res[0]) {
-		res[0].category = JSON.parse(res[0].category)
-		res[0].owners = JSON.parse(res[0].owners)
-		if (owners)
-			res[0].owners = await Promise.all(
-				res[0].owners.map(async (u: string) => await userWithNoBot.load(u))
-			)
-		res[0].owners = res[0].owners.filter((el: User | null) => el).map((row: User) => ({ ...row }))
-		res[0].vanity = ((res[0].trusted || res[0].partnered) && res[0].vanity) ?? null
-	}
-
-	return res[0] || null
-}
-
-async function getUser(id: string, bots = true): Promise<User> {
-	const res = await knex('users')
-		.select(['id', 'avatar', 'tag', 'username', 'perm', 'github'])
-		.where({ id })
-	if (res[0]) {
-		const owned = await knex('bots')
-			.select(['id'])
-			.where('owners', 'like', `%${id}%`)
-		if (bots) res[0].bots = await Promise.all(owned.map(async b => await botWithNoUser.load(b.id)))
-		else res[0].bots = owned.map(async b => b.id)
-		res[0].bots = res[0].bots.filter((el: Bot | null) => el).map((row: User) => ({ ...row }))
-	}
-
-	return res[0] || null
-}
-
-async function getBotList(type: ListType, page = 1, query?: string):Promise<BotList> {
-	let res: { id: string }[]
-	let count: ReactText
-	if (type === 'VOTE') {
-		count = (await knex('bots').count())[0]['count(*)']
-		res = await knex('bots')
-			.orderBy('votes', 'desc')
-			.orderBy('servers', 'desc')
-			.limit(16)
-			.offset(((page ? Number(page) : 1) - 1) * 16)
-			.select(['id'])
-	} else if (type === 'TRUSTED') {
-		count = (
-			await knex('bots')
-				.where({ trusted: true })
-				.count()
-		)[0]['count(*)']
-		res = await knex('bots')
-			.where({ trusted: true })
-			.orderByRaw('RAND()')
-			.limit(16)
-			.offset(((page ? Number(page) : 1) - 1) * 16)
-			.select(['id'])
-	} else if (type === 'NEW') {
-		count = (
-			await knex('bots')
-				.count()
-		)[0]['count(*)']
-		res = await knex('bots')
-			.orderBy('date', 'desc')
-			.limit(16)
-			.offset(((page ? Number(page) : 1) - 1) * 16)
-			.select(['id'])
-	} else if (type === 'PARTNERED') {
-		count = (
-			await knex('bots')
-				.where({ partnered: true })
-				.count()
-		)[0]['count(*)']
-		res = await knex('bots')
-			.where({ partnered: true })
-			.orderByRaw('RAND()')
-			.limit(16)
-			.offset(((page ? Number(page) : 1) - 1) * 16)
-			.select(['id'])
-	} else if (type === 'CATEGORY') {
-		if (!query) throw new Error('쿼리가 누락되었습니다.')
-		if (!cats.includes(query)) throw new Error('알 수 없는 카테고리입니다.')
-		count = (
-			await knex('bots')
-				.where('category', 'like', `%${decodeURI(query)}%`)
-				.count()
-		)[0]['count(*)']
-		res = await knex('bots')
-			.where('category', 'like', `%${decodeURI(query)}%`)
-			.orderBy('votes', 'desc')
-			.orderBy('servers', 'desc')
-			.limit(16)
-			.offset(((page ? Number(page) : 1) - 1) * 16)
-			.select(['id'])
-	} else if (type === 'SEARCH') {
-		if (!query) throw new Error('쿼리가 누락되었습니다.')
-		try {
-			new RegExp(query)
-			count = (
-				await knex('bots')
-					.where('name', 'REGEXP', query)
-					.orWhere('intro', 'REGEXP', query)
-					.orWhere('desc', 'REGEXP', query)
-					.orderBy('votes', 'desc')
-					.orderBy('servers', 'desc')
-					.count()
-			)[0]['count(*)']
-
-			res = await knex('bots')
-				.where('name', 'REGEXP', query)
-				.orWhere('intro', 'REGEXP', query)
-				.orWhere('desc', 'REGEXP', query)
-				.orderBy('votes', 'desc')
-				.orderBy('servers', 'desc')
-				.limit(16)
-				.offset(((page ? Number(page) : 1) - 1) * 16)
-				.select(['id'])
-		} catch (e) {
-			count = (
-				await knex('bots')
-					.where('name', 'LIKE', `%${query}%`)
-					.orWhere('intro', 'LIKE', `%${query}%`)
-					.orWhere('desc', 'LIKE', `%${query}%`)
-					.orderBy('votes', 'desc')
-					.orderBy('servers', 'desc')
-					.count()
-			)[0]['count(*)']
-
-			res = await knex('bots')
-				.where('name', 'LIKE', `%${query}%`)
-				.orWhere('intro', 'LIKE', `%${query}%`)
-				.orWhere('desc', 'LIKE', `%${query}%`)
-				.orderBy('votes', 'desc')
-				.orderBy('servers', 'desc')
-				.limit(16)
-				.offset(((page ? Number(page) : 1) - 1) * 16)
-				.select(['id'])
-		}
-	} else {
-		count = 1
-		res = []
-	}
-
-	return { type, data: (await Promise.all(res.map(async el => await getBot(el.id)))).map(r=> ({...r})), currentPage: page, totalPage: Math.ceil(Number(count) / 16) }
-}
-
-export default { bot, user, botListVotes, botListNew, botListTrusted }

utils/Query.ts

@@ -1,15 +1,18 @@
 import fetch from 'node-fetch'
 import jwt from 'jsonwebtoken'
 import fs from 'fs'
+import { TLRU } from 'tlru'
 
-import { Bot, User } from '../types'
+import { Bot, User, ListType, BotList } from '../types'
+import { cats } from './Constants'
 
 import knex from './Knex'
+import DataLoader from 'dataloader'
 
 const publicPem = fs.readFileSync('./public.pem')
 const privateKey = fs.readFileSync('./private.key')
 
-export async function getBot(id: string, owners = true): Promise<Bot> {
+async function getBot(id: string, owners=true) {
 	const res = await knex('bots')
 		.select([
 			'id',
@@ -39,20 +42,23 @@ export async function getBot(id: string, owners = true): Promise<Bot> {
 			'banner',
 		])
 		.where({ id })
-		.orWhere({ vanity: id, boosted: 1 })
+		.orWhere({ vanity: id, trusted: true })
+		.orWhere({ vanity: id, partnered: true })
 	if (res[0]) {
 		res[0].category = JSON.parse(res[0].category)
 		res[0].owners = JSON.parse(res[0].owners)
-		if (owners)
+		res[0].vanity = ((res[0].trusted || res[0].partnered) && res[0].vanity) ?? null
-			res[0].owners = await Promise.all(res[0].owners.map(async (u: string) => await getUser(u)))
-		res[0].owners = res[0].owners.filter((el: User | null) => el).map((row: User) => ({ ...row }))
-		res[0].vanity = res[0].vanity && (res[0].boosted || res[0].trusted || res[0].partnered)
 	}
+	if (owners)
+		res[0].owners = await Promise.all(
+			res[0].owners.map(async (u: string) => await get._rawUser.load(u))
+		)
+	res[0].owners = res[0].owners.filter((el: User | null) => el).map((row: User) => ({ ...row }))
 
-	return res[0] || null
+	return res[0] ?? null
 }
 
-export async function getUser(id: string, bots = true): Promise<User> {
+async function getUser(id: string, bots = true) {
 	const res = await knex('users')
 		.select(['id', 'avatar', 'tag', 'username', 'perm', 'github'])
 		.where({ id })
@@ -60,10 +66,155 @@ export async function getUser(id: string, bots = true): Promise<User> {
 		const owned = await knex('bots')
 			.select(['id'])
 			.where('owners', 'like', `%${id}%`)
-		if (bots) res[0].bots = await Promise.all(owned.map(async b => await getBot(b.id, false)))
+		if (bots) res[0].bots = await Promise.all(owned.map(async b => await get._rawBot.load(b.id)))
 		else res[0].bots = owned.map(async b => b.id)
 		res[0].bots = res[0].bots.filter((el: Bot | null) => el).map((row: User) => ({ ...row }))
 	}
 
 	return res[0] || null
 }
+
+async function getBotList(type: ListType, page = 1, query?: string):Promise<BotList> {
+	let res: { id: string }[]
+	let count:string|number
+	if (type === 'VOTE') {
+		count = (await knex('bots').count())[0]['count(*)']
+		res = await knex('bots')
+			.orderBy('votes', 'desc')
+			.orderBy('servers', 'desc')
+			.limit(16)
+			.offset(((page ? Number(page) : 1) - 1) * 16)
+			.select(['id'])
+	} else if (type === 'TRUSTED') {
+		count = (
+			await knex('bots')
+				.where({ trusted: true })
+				.count()
+		)[0]['count(*)']
+		res = await knex('bots')
+			.where({ trusted: true })
+			.orderByRaw('RAND()')
+			.limit(16)
+			.offset(((page ? Number(page) : 1) - 1) * 16)
+			.select(['id'])
+	} else if (type === 'NEW') {
+		count = (
+			await knex('bots')
+				.count()
+		)[0]['count(*)']
+		res = await knex('bots')
+			.orderBy('date', 'desc')
+			.limit(16)
+			.offset(((page ? Number(page) : 1) - 1) * 16)
+			.select(['id'])
+	} else if (type === 'PARTNERED') {
+		count = (
+			await knex('bots')
+				.where({ partnered: true })
+				.count()
+		)[0]['count(*)']
+		res = await knex('bots')
+			.where({ partnered: true })
+			.orderByRaw('RAND()')
+			.limit(16)
+			.offset(((page ? Number(page) : 1) - 1) * 16)
+			.select(['id'])
+	} else if (type === 'CATEGORY') {
+		if (!query) throw new Error('쿼리가 누락되었습니다.')
+		if (!cats.includes(query)) throw new Error('알 수 없는 카테고리입니다.')
+		count = (
+			await knex('bots')
+				.where('category', 'like', `%${decodeURI(query)}%`)
+				.count()
+		)[0]['count(*)']
+		res = await knex('bots')
+			.where('category', 'like', `%${decodeURI(query)}%`)
+			.orderBy('votes', 'desc')
+			.orderBy('servers', 'desc')
+			.limit(16)
+			.offset(((page ? Number(page) : 1) - 1) * 16)
+			.select(['id'])
+	} else if (type === 'SEARCH') {
+		if (!query) throw new Error('쿼리가 누락되었습니다.')
+		try {
+			new RegExp(query)
+			count = (
+				await knex('bots')
+					.where('name', 'REGEXP', query)
+					.orWhere('intro', 'REGEXP', query)
+					.orWhere('desc', 'REGEXP', query)
+					.orderBy('votes', 'desc')
+					.orderBy('servers', 'desc')
+					.count()
+			)[0]['count(*)']
+
+			res = await knex('bots')
+				.where('name', 'REGEXP', query)
+				.orWhere('intro', 'REGEXP', query)
+				.orWhere('desc', 'REGEXP', query)
+				.orderBy('votes', 'desc')
+				.orderBy('servers', 'desc')
+				.limit(16)
+				.offset(((page ? Number(page) : 1) - 1) * 16)
+				.select(['id'])
+		} catch (e) {
+			count = (
+				await knex('bots')
+					.where('name', 'LIKE', `%${query}%`)
+					.orWhere('intro', 'LIKE', `%${query}%`)
+					.orWhere('desc', 'LIKE', `%${query}%`)
+					.orderBy('votes', 'desc')
+					.orderBy('servers', 'desc')
+					.count()
+			)[0]['count(*)']
+
+			res = await knex('bots')
+				.where('name', 'LIKE', `%${query}%`)
+				.orWhere('intro', 'LIKE', `%${query}%`)
+				.orWhere('desc', 'LIKE', `%${query}%`)
+				.orderBy('votes', 'desc')
+				.orderBy('servers', 'desc')
+				.limit(16)
+				.offset(((page ? Number(page) : 1) - 1) * 16)
+				.select(['id'])
+		}
+	} else {
+		count = 1
+		res = []
+	}
+
+	return { type, data: (await Promise.all(res.map(async el => await getBot(el.id)))).map(r=> ({...r})), currentPage: page, totalPage: Math.ceil(Number(count) / 16) }
+}
+
+export const get = {
+	bot: new DataLoader(
+		async (ids: string[]) =>
+			(await Promise.all(ids.map(async (el: string) => await getBot(el)))).map(row => ({ ...row }))
+		, { cacheMap: new TLRU({ maxStoreSize: 50, maxAgeMs: 60000 }) }),
+	_rawBot: new DataLoader(
+		async (ids: string[]) =>
+			(await Promise.all(ids.map(async (el: string) => await getBot(el, false)))).map(row => ({ ...row }))
+		, { cacheMap: new TLRU({ maxStoreSize: 50, maxAgeMs: 60000 }) }),
+	user: new DataLoader(
+		async (ids: string[]) =>
+			(await Promise.all(ids.map(async (el: string) => await getUser(el)))).map(row => ({ ...row }))
+		, { cacheMap: new TLRU({ maxStoreSize: 50, maxAgeMs: 60000 }) }),
+	_rawUser: new DataLoader(
+		async (ids: string[]) =>
+			(await Promise.all(ids.map(async (el: string) => await getUser(el, false)))).map(row => ({ ...row }))
+		, { cacheMap: new TLRU({ maxStoreSize: 50, maxAgeMs: 60000 }) }),
+	list: {
+		votes: new DataLoader(
+			async (pages: number[]) =>
+				(await Promise.all(pages.map(async (page: number) => await getBotList('VOTE', page)))).map(row => ({ ...row }))
+			, { cacheMap: new TLRU({ maxStoreSize: 50, maxAgeMs: 600000 }) }),
+		new: new DataLoader(
+			async (pages: number[]) =>
+				(await Promise.all(pages.map(async (page: number) => await getBotList('NEW', page)))).map(row => ({ ...row }))
+			, { cacheMap: new TLRU({ maxStoreSize: 50, maxAgeMs: 1800000 }) }),
+		trusted: new DataLoader(
+			async (pages: number[]) =>
+				(await Promise.all(pages.map(async (page: number) => await getBotList('TRUSTED', page)))).map(row => ({ ...row }))
+			, { cacheMap: new TLRU({ maxStoreSize: 50, maxAgeMs: 3600000 }) }),
+	}
+}

utils/index.ts

@@ -1,6 +1,5 @@
 import * as Query from './Query'
 import {  } from './Tools'
 import ResponseWrapper from './ResponseWrapper'
-import Fetch from './Fetch'
 
-export { Query, Fetch, ResponseWrapper }
+export { Query, ResponseWrapper }