wonderlandpark/core
2
1
Fork 0
mirror of https://github.com/koreanbots/core.git synced 2025-12-15 14:10:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: added tlru caching

Browse Source 
close: https://github.com/koreanbots/v2-testing/issues/22
This commit is contained in:
원더 2021-01-11 15:33:33 +09:00
parent 5317e4a053
commit f4c3847538
6 changed files with 174 additions and 255 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
pages/bots/[id].tsx
View File

@ -14,13 +14,13 @@ import NotFound from '../404'
import SEO from '../../components/SEO'
import LongButton from '../../components/LongButton'
import { git, Status } from '../../utils/Constants'
import { Fetch } from '../../utils'
import { Query } from '../../utils'
import { formatNumber } from '../../utils/Tools'
import Advertisement from '../../components/Advertisement'
import Link from 'next/link'
const Bots: NextPage<BotsProps> = ({ data, date }) => {
if (!data.id) return <NotFound />
if (!data || !data.id) return <NotFound />
return (
<Container paddingTop className='py-10'>
<SEO
@ -208,11 +208,11 @@ const Bots: NextPage<BotsProps> = ({ data, date }) => {
}
export const getServerSideProps = async (ctx: Context) => {
const data = await Fetch.bot.load(ctx.query.id)
const data = await Query.get.bot.load(ctx.query.id) ?? {}
return {
props: {
data,
date: SnowflakeUtil.deconstruct(data.id ?? '0').date.toJSON()
date: SnowflakeUtil.deconstruct(data?.id ?? '0').date.toJSON()
},
}
}
@ -225,9 +225,9 @@ interface BotsProps {
votes: string
}
interface Context extends NextPageContext {
query: Query
query: URLQuery
}
interface Query extends ParsedUrlQuery {
interface URLQuery extends ParsedUrlQuery {
id: string
}

8
pages/index.tsx
View File

@ -5,7 +5,7 @@ import Container from '../components/Container'
import LongButton from '../components/LongButton'
import Wave from '../components/Wave'
import { BotList } from '../types'
import { Fetch } from '../utils'
import { Query } from '../utils'
const Index: NextPage<IndexProps> = ({ votes, newBots, trusted }) => {
return (
@ -61,9 +61,9 @@ const Index: NextPage<IndexProps> = ({ votes, newBots, trusted }) => {
}
export const getServerSideProps = async() => {
const votes = await Fetch.botListVotes.load(1)
const newBots = await Fetch.botListNew.load(1)
const trusted = await Fetch.botListTrusted.load(1)
const votes = await Query.get.list.votes.load(1)
const newBots = await Query.get.list.new.load(1)
const trusted = await Query.get.list.trusted.load(1)
return { props: { votes, newBots, trusted }}

4
pages/users/[id].tsx
View File

@ -1,6 +1,6 @@
import { NextPage, NextPageContext } from 'next'
import { SnowflakeUtil } from 'discord.js'
import { Fetch } from '../../utils'
import { Query } from '../../utils'
import { ParsedUrlQuery } from 'querystring'
import { josa } from 'josa'
import { Bot, User } from '../../types'
@ -88,7 +88,7 @@ const Users: NextPage<UserProps> = ({ data }) => {
}
export const getServerSideProps = async (ctx: Context) => {
const data = await Fetch.user.load(ctx.query.id)
const data = await Query.get.user.load(ctx.query.id)
return { props: { data, date: SnowflakeUtil.deconstruct(data.id ?? '0')?.date?.toJSON() } }
}

231
utils/Fetch.ts
View File

@ -1,231 +0,0 @@
import DataLoader from 'dataloader'
import fs from 'fs'
import { Bot, ListType, User, BotList } from '../types'
import knex from './Knex'
import { cats } from './Constants'
import { botListArgument } from './Yup'
import { ReactText } from 'react'
const publicPem = fs.readFileSync('./public.pem')
const privateKey = fs.readFileSync('./private.key')
const bot = new DataLoader(
async (ids: string[]) =>
(await Promise.all(ids.map(async (el: string) => getBot(el)))).map(row => ({ ...row }))
)
new DataLoader(async ()=> { return []})
const botWithNoUser = new DataLoader(
async (ids: string[]) =>
(await Promise.all(ids.map(async (el: string) => getBot(el, false)))).map(row => ({ ...row }))
)
const user = new DataLoader(
async (ids: string[]) =>
(await Promise.all(ids.map((el: string) => getUser(el)))).map(row => ({ ...row }))
)
const userWithNoBot = new DataLoader(
async (ids: string[]) =>
(await Promise.all(ids.map((el: string) => getUser(el, false)))).map(row => ({ ...row }))
)
const botListVotes = new DataLoader(
async (page: number[]) =>
(await Promise.all(page.map((el) => getBotList('VOTE', el)))).map(row => ({ ...row })),
{
batchScheduleFn: callback => setTimeout(callback, 100),
}
)
const botListNew = new DataLoader(
async (page: number[]) =>
(await Promise.all(page.map((el) => getBotList('NEW', el)))).map(row => ({ ...row })),
{
batchScheduleFn: callback => setTimeout(callback, 100),
}
)
const botListTrusted = new DataLoader(
async (page: number[]) =>
(await Promise.all(page.map((el) => getBotList('TRUSTED', el)))).map(row => ({ ...row })),
{
batchScheduleFn: callback => setTimeout(callback, 100),
}
)
async function getBot(id: string, owners = true): Promise<Bot> {
const res = await knex('bots')
.select([
'id',
'owners',
'lib',
'prefix',
'votes',
'servers',
'intro',
'desc',
'web',
'git',
'url',
'category',
'status',
'name',
'avatar',
'tag',
'verified',
'trusted',
'partnered',
'discord',
'boosted',
'state',
'vanity',
'bg',
'banner',
])
.where({ id })
.orWhere({ vanity: id, trusted: true })
.orWhere({ vanity: id, partnered: true })
if (res[0]) {
res[0].category = JSON.parse(res[0].category)
res[0].owners = JSON.parse(res[0].owners)
if (owners)
res[0].owners = await Promise.all(
res[0].owners.map(async (u: string) => await userWithNoBot.load(u))
)
res[0].owners = res[0].owners.filter((el: User | null) => el).map((row: User) => ({ ...row }))
res[0].vanity = ((res[0].trusted || res[0].partnered) && res[0].vanity) ?? null
}
return res[0] || null
}
async function getUser(id: string, bots = true): Promise<User> {
const res = await knex('users')
.select(['id', 'avatar', 'tag', 'username', 'perm', 'github'])
.where({ id })
if (res[0]) {
const owned = await knex('bots')
.select(['id'])
.where('owners', 'like', `%${id}%`)
if (bots) res[0].bots = await Promise.all(owned.map(async b => await botWithNoUser.load(b.id)))
else res[0].bots = owned.map(async b => b.id)
res[0].bots = res[0].bots.filter((el: Bot | null) => el).map((row: User) => ({ ...row }))
}
return res[0] || null
}
async function getBotList(type: ListType, page = 1, query?: string):Promise<BotList> {
let res: { id: string }[]
let count: ReactText
if (type === 'VOTE') {
count = (await knex('bots').count())[0]['count(*)']
res = await knex('bots')
.orderBy('votes', 'desc')
.orderBy('servers', 'desc')
.limit(16)
.offset(((page ? Number(page) : 1) - 1) * 16)
.select(['id'])
} else if (type === 'TRUSTED') {
count = (
await knex('bots')
.where({ trusted: true })
.count()
)[0]['count(*)']
res = await knex('bots')
.where({ trusted: true })
.orderByRaw('RAND()')
.limit(16)
.offset(((page ? Number(page) : 1) - 1) * 16)
.select(['id'])
} else if (type === 'NEW') {
count = (
await knex('bots')
.count()
)[0]['count(*)']
res = await knex('bots')
.orderBy('date', 'desc')
.limit(16)
.offset(((page ? Number(page) : 1) - 1) * 16)
.select(['id'])
} else if (type === 'PARTNERED') {
count = (
await knex('bots')
.where({ partnered: true })
.count()
)[0]['count(*)']
res = await knex('bots')
.where({ partnered: true })
.orderByRaw('RAND()')
.limit(16)
.offset(((page ? Number(page) : 1) - 1) * 16)
.select(['id'])
} else if (type === 'CATEGORY') {
if (!query) throw new Error('쿼리가 누락되었습니다.')
if (!cats.includes(query)) throw new Error('알 수 없는 카테고리입니다.')
count = (
await knex('bots')
.where('category', 'like', `%${decodeURI(query)}%`)
.count()
)[0]['count(*)']
res = await knex('bots')
.where('category', 'like', `%${decodeURI(query)}%`)
.orderBy('votes', 'desc')
.orderBy('servers', 'desc')
.limit(16)
.offset(((page ? Number(page) : 1) - 1) * 16)
.select(['id'])
} else if (type === 'SEARCH') {
if (!query) throw new Error('쿼리가 누락되었습니다.')
try {
new RegExp(query)
count = (
await knex('bots')
.where('name', 'REGEXP', query)
.orWhere('intro', 'REGEXP', query)
.orWhere('desc', 'REGEXP', query)
.orderBy('votes', 'desc')
.orderBy('servers', 'desc')
.count()
)[0]['count(*)']
res = await knex('bots')
.where('name', 'REGEXP', query)
.orWhere('intro', 'REGEXP', query)
.orWhere('desc', 'REGEXP', query)
.orderBy('votes', 'desc')
.orderBy('servers', 'desc')
.limit(16)
.offset(((page ? Number(page) : 1) - 1) * 16)
.select(['id'])
} catch (e) {
count = (
await knex('bots')
.where('name', 'LIKE', `%${query}%`)
.orWhere('intro', 'LIKE', `%${query}%`)
.orWhere('desc', 'LIKE', `%${query}%`)
.orderBy('votes', 'desc')
.orderBy('servers', 'desc')
.count()
)[0]['count(*)']
res = await knex('bots')
.where('name', 'LIKE', `%${query}%`)
.orWhere('intro', 'LIKE', `%${query}%`)
.orWhere('desc', 'LIKE', `%${query}%`)
.orderBy('votes', 'desc')
.orderBy('servers', 'desc')
.limit(16)
.offset(((page ? Number(page) : 1) - 1) * 16)
.select(['id'])
}
} else {
count = 1
res = []
}
return { type, data: (await Promise.all(res.map(async el => await getBot(el.id)))).map(r=> ({...r})), currentPage: page, totalPage: Math.ceil(Number(count) / 16) }
}
export default { bot, user, botListVotes, botListNew, botListTrusted }

171
utils/Query.ts
View File

@ -1,15 +1,18 @@
import fetch from 'node-fetch'
import jwt from 'jsonwebtoken'
import fs from 'fs'
import { TLRU } from 'tlru'
import { Bot, User } from '../types'
import { Bot, User, ListType, BotList } from '../types'
import { cats } from './Constants'
import knex from './Knex'
import DataLoader from 'dataloader'
const publicPem = fs.readFileSync('./public.pem')
const privateKey = fs.readFileSync('./private.key')
export async function getBot(id: string, owners = true): Promise<Bot> {
async function getBot(id: string, owners=true) {
const res = await knex('bots')
.select([
'id',
@ -39,20 +42,23 @@ export async function getBot(id: string, owners = true): Promise<Bot> {
'banner',
])
.where({ id })
.orWhere({ vanity: id, boosted: 1 })
.orWhere({ vanity: id, trusted: true })
.orWhere({ vanity: id, partnered: true })
if (res[0]) {
res[0].category = JSON.parse(res[0].category)
res[0].owners = JSON.parse(res[0].owners)
if (owners)
res[0].owners = await Promise.all(res[0].owners.map(async (u: string) => await getUser(u)))
res[0].owners = res[0].owners.filter((el: User | null) => el).map((row: User) => ({ ...row }))
res[0].vanity = res[0].vanity && (res[0].boosted || res[0].trusted || res[0].partnered)
res[0].vanity = ((res[0].trusted || res[0].partnered) && res[0].vanity) ?? null
}
if (owners)
res[0].owners = await Promise.all(
res[0].owners.map(async (u: string) => await get._rawUser.load(u))
)
res[0].owners = res[0].owners.filter((el: User | null) => el).map((row: User) => ({ ...row }))
return res[0] || null
return res[0] ?? null
}
export async function getUser(id: string, bots = true): Promise<User> {
async function getUser(id: string, bots = true) {
const res = await knex('users')
.select(['id', 'avatar', 'tag', 'username', 'perm', 'github'])
.where({ id })
@ -60,10 +66,155 @@ export async function getUser(id: string, bots = true): Promise<User> {
const owned = await knex('bots')
.select(['id'])
.where('owners', 'like', `%${id}%`)
if (bots) res[0].bots = await Promise.all(owned.map(async b => await getBot(b.id, false)))
if (bots) res[0].bots = await Promise.all(owned.map(async b => await get._rawBot.load(b.id)))
else res[0].bots = owned.map(async b => b.id)
res[0].bots = res[0].bots.filter((el: Bot | null) => el).map((row: User) => ({ ...row }))
}
return res[0] || null
}
async function getBotList(type: ListType, page = 1, query?: string):Promise<BotList> {
let res: { id: string }[]
let count:string|number
if (type === 'VOTE') {
count = (await knex('bots').count())[0]['count(*)']
res = await knex('bots')
.orderBy('votes', 'desc')
.orderBy('servers', 'desc')
.limit(16)
.offset(((page ? Number(page) : 1) - 1) * 16)
.select(['id'])
} else if (type === 'TRUSTED') {
count = (
await knex('bots')
.where({ trusted: true })
.count()
)[0]['count(*)']
res = await knex('bots')
.where({ trusted: true })
.orderByRaw('RAND()')
.limit(16)
.offset(((page ? Number(page) : 1) - 1) * 16)
.select(['id'])
} else if (type === 'NEW') {
count = (
await knex('bots')
.count()
)[0]['count(*)']
res = await knex('bots')
.orderBy('date', 'desc')
.limit(16)
.offset(((page ? Number(page) : 1) - 1) * 16)
.select(['id'])
} else if (type === 'PARTNERED') {
count = (
await knex('bots')
.where({ partnered: true })
.count()
)[0]['count(*)']
res = await knex('bots')
.where({ partnered: true })
.orderByRaw('RAND()')
.limit(16)
.offset(((page ? Number(page) : 1) - 1) * 16)
.select(['id'])
} else if (type === 'CATEGORY') {
if (!query) throw new Error('쿼리가 누락되었습니다.')
if (!cats.includes(query)) throw new Error('알 수 없는 카테고리입니다.')
count = (
await knex('bots')
.where('category', 'like', `%${decodeURI(query)}%`)
.count()
)[0]['count(*)']
res = await knex('bots')
.where('category', 'like', `%${decodeURI(query)}%`)
.orderBy('votes', 'desc')
.orderBy('servers', 'desc')
.limit(16)
.offset(((page ? Number(page) : 1) - 1) * 16)
.select(['id'])
} else if (type === 'SEARCH') {
if (!query) throw new Error('쿼리가 누락되었습니다.')
try {
new RegExp(query)
count = (
await knex('bots')
.where('name', 'REGEXP', query)
.orWhere('intro', 'REGEXP', query)
.orWhere('desc', 'REGEXP', query)
.orderBy('votes', 'desc')
.orderBy('servers', 'desc')
.count()
)[0]['count(*)']
res = await knex('bots')
.where('name', 'REGEXP', query)
.orWhere('intro', 'REGEXP', query)
.orWhere('desc', 'REGEXP', query)
.orderBy('votes', 'desc')
.orderBy('servers', 'desc')
.limit(16)
.offset(((page ? Number(page) : 1) - 1) * 16)
.select(['id'])
} catch (e) {
count = (
await knex('bots')
.where('name', 'LIKE', `%${query}%`)
.orWhere('intro', 'LIKE', `%${query}%`)
.orWhere('desc', 'LIKE', `%${query}%`)
.orderBy('votes', 'desc')
.orderBy('servers', 'desc')
.count()
)[0]['count(*)']
res = await knex('bots')
.where('name', 'LIKE', `%${query}%`)
.orWhere('intro', 'LIKE', `%${query}%`)
.orWhere('desc', 'LIKE', `%${query}%`)
.orderBy('votes', 'desc')
.orderBy('servers', 'desc')
.limit(16)
.offset(((page ? Number(page) : 1) - 1) * 16)
.select(['id'])
}
} else {
count = 1
res = []
}
return { type, data: (await Promise.all(res.map(async el => await getBot(el.id)))).map(r=> ({...r})), currentPage: page, totalPage: Math.ceil(Number(count) / 16) }
}
export const get = {
bot: new DataLoader(
async (ids: string[]) =>
(await Promise.all(ids.map(async (el: string) => await getBot(el)))).map(row => ({ ...row }))
, { cacheMap: new TLRU({ maxStoreSize: 50, maxAgeMs: 60000 }) }),
_rawBot: new DataLoader(
async (ids: string[]) =>
(await Promise.all(ids.map(async (el: string) => await getBot(el, false)))).map(row => ({ ...row }))
, { cacheMap: new TLRU({ maxStoreSize: 50, maxAgeMs: 60000 }) }),
user: new DataLoader(
async (ids: string[]) =>
(await Promise.all(ids.map(async (el: string) => await getUser(el)))).map(row => ({ ...row }))
, { cacheMap: new TLRU({ maxStoreSize: 50, maxAgeMs: 60000 }) }),
_rawUser: new DataLoader(
async (ids: string[]) =>
(await Promise.all(ids.map(async (el: string) => await getUser(el, false)))).map(row => ({ ...row }))
, { cacheMap: new TLRU({ maxStoreSize: 50, maxAgeMs: 60000 }) }),
list: {
votes: new DataLoader(
async (pages: number[]) =>
(await Promise.all(pages.map(async (page: number) => await getBotList('VOTE', page)))).map(row => ({ ...row }))
, { cacheMap: new TLRU({ maxStoreSize: 50, maxAgeMs: 600000 }) }),
new: new DataLoader(
async (pages: number[]) =>
(await Promise.all(pages.map(async (page: number) => await getBotList('NEW', page)))).map(row => ({ ...row }))
, { cacheMap: new TLRU({ maxStoreSize: 50, maxAgeMs: 1800000 }) }),
trusted: new DataLoader(
async (pages: number[]) =>
(await Promise.all(pages.map(async (page: number) => await getBotList('TRUSTED', page)))).map(row => ({ ...row }))
, { cacheMap: new TLRU({ maxStoreSize: 50, maxAgeMs: 3600000 }) }),
}
}

3
utils/index.ts
View File

@ -1,6 +1,5 @@
import * as Query from './Query'
import { } from './Tools'
import ResponseWrapper from './ResponseWrapper'
import Fetch from './Fetch'
export { Query, Fetch, ResponseWrapper }
export { Query, ResponseWrapper }